luna - wolfssl/wolfcrypt/src/port/riscv/riscv-64-aes.c

   1/* riscv-64-aes.c
   2 *
   3 * Copyright (C) 2006-2026 wolfSSL Inc.
   4 *
   5 * This file is part of wolfSSL.
   6 *
   7 * wolfSSL is free software; you can redistribute it and/or modify
   8 * it under the terms of the GNU General Public License as published by
   9 * the Free Software Foundation; either version 3 of the License, or
  10 * (at your option) any later version.
  11 *
  12 * wolfSSL is distributed in the hope that it will be useful,
  13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 * GNU General Public License for more details.
  16 *
  17 * You should have received a copy of the GNU General Public License
  18 * along with this program; if not, write to the Free Software
  19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
  20 */
  21
  22#include <wolfssl/wolfcrypt/libwolfssl_sources.h>
  23
  24#if FIPS_VERSION3_GE(2,0,0)
  25    /* set NO_WRAPPERS before headers, use direct internal f()s not wrappers */
  26    #define FIPS_NO_WRAPPERS
  27#endif
  28
  29#include <wolfssl/wolfcrypt/port/riscv/riscv-64-asm.h>
  30
  31#if !defined(NO_AES)
  32
  33#include <wolfssl/wolfcrypt/aes.h>
  34
  35#ifdef NO_INLINE
  36    #include <wolfssl/wolfcrypt/misc.h>
  37#else
  38    #define WOLFSSL_MISC_INCLUDED
  39    #include <wolfcrypt/src/misc.c>
  40#endif
  41
  42#ifdef WOLFSSL_RISCV_ASM
  43
  44#if FIPS_VERSION3_GE(6,0,0)
  45    const unsigned int wolfCrypt_FIPS_aes_ro_sanity[2] =
  46                                                     { 0x1a2b3c4d, 0x00000002 };
  47    int wolfCrypt_FIPS_AES_sanity(void)
  48    {
  49        return 0;
  50    }
  51#endif
  52
  53/* Copy a 16-byte value from in to out.
  54 *
  55 * @param [out] out  16-byte value destination.
  56 * @param [in]  in   16-byte value source.
  57 */
  58static WC_INLINE void memcpy16(byte* out, const byte* in)
  59{
  60    word64* out64 = (word64*)out;
  61    word64* in64  = (word64*)in;
  62
  63    out64[0] = in64[0];
  64    out64[1] = in64[1];
  65}
  66
  67#ifdef WOLFSSL_RISCV_BIT_MANIPULATION
  68
  69/* Reverse bits in each byte of 64-bit register. */
  70#define BREV8(rd, rs)                                       \
  71    ASM_WORD(0b01101000011100000101000000010011 |           \
  72             (rs << 15) | (rd << 7))
  73
  74#endif /* WOLFSSL_RISCV_BIT_MANIPULATION */
  75
  76#ifdef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
  77
  78/* Reverse bits in each byte of 128-bit vector register. */
  79#define VBREV8(vd, vs2) \
  80    ASM_WORD((0b010010 << 26) | (0b1 << 25) | (0b1000 << 15) | \
  81             (0b010 << 12) | (0b1010111 << 0) | \
  82             (vs2 << 20) | (vd << 7))
  83#endif
  84
  85
  86/* Vector register set if equal: vd[i] = vs1[i] == vs2[i] ? 1 : 0 */
  87#define VMSEQ_VV(vd, vs1, vs2)                      \
  88    ASM_WORD((0b011000 << 26) | (0b1 << 25) |       \
  89             (0b000 << 12) | (0b1010111 << 0) |     \
  90             (vs2 << 20) | (vs1 << 15) | (vd << 7))
  91/* Vector register set if equal: vd[i] = vs1[i] != vs2[i] ? 1 : 0 */
  92#define VMSNE_VV(vd, vs1, vs2)                      \
  93    ASM_WORD((0b011001 << 26) | (0b1 << 25) |       \
  94             (0b000 << 12) | (0b1010111 << 0) |     \
  95             (vs2 << 20) | (vs1 << 15) | (vd << 7))
  96
  97/* rd = Count of vs2[i] that has a value of 1. */
  98#define VCPOP_M(rd, vs2)                            \
  99    ASM_WORD((0b010000 << 26) | (0b1 << 25) |       \
 100             (0b10000 << 15) |                      \
 101             (0b010 << 12) | (0b1010111 << 0) |     \
 102             (vs2 << 20) | (rd << 7))
 103
 104#if defined(WOLFSSL_RISCV_VECTOR_CRYPTO_ASM)
 105
 106/*
 107 * Vector crypto instruction implementation of base operations.
 108 */
 109
 110/* Vector AES-128 forward key schedule computation. */
 111#define VAESKF1_VI(rd, rs2, rnum)                   \
 112    ASM_WORD((0b100010 << 26) | (0b1 << 25) |       \
 113             (0b010 << 12) | (0b1110111 << 0) |     \
 114             (rd << 7) | (rnum << 15) | (rs2 << 20))
 115/* Vector AES-256 forward key schedule computation. */
 116#define VAESKF2_VI(rd, rs2, rnum)                   \
 117    ASM_WORD((0b101010 << 26) | (0b1 << 25) |       \
 118             (0b010 << 12) | (0b1110111 << 0) |     \
 119             (rd << 7) | (rnum << 15) | (rs2 << 20))
 120
 121/* Vector AES round zero encryption/decryption. */
 122#define VAESZ_VS(rd, rs2)                           \
 123    ASM_WORD((0b101001 << 26) | (0b1 << 25) |       \
 124             (0b00111 << 15) | (0b010 << 12) |      \
 125             (0b1110111 << 0) |                     \
 126             (rd << 7) | (rs2 << 20))
 127/* Vector AES middle-round encryption. */
 128#define VAESEM_VS(rd, rs2)                          \
 129    ASM_WORD((0b101001 << 26) | (0b1 << 25) |       \
 130             (0b00010 << 15) | (0b010 << 12) |      \
 131             (0b1110111 << 0) |                     \
 132             (rd << 7) | (rs2 << 20))
 133/* Vector AES final-round encryption. */
 134#define VAESEF_VS(rd, rs2)                          \
 135    ASM_WORD((0b101001 << 26) | (0b1 << 25) |       \
 136             (0b00011 << 15) | (0b010 << 12) |      \
 137             (0b1110111 << 0) |                     \
 138             (rd << 7) | (rs2 << 20))
 139/* Vector AES middle-round decryption. */
 140#define VAESDM_VS(rd, rs2)                          \
 141    ASM_WORD((0b101001 << 26) | (0b1 << 25) |       \
 142             (0b00000 << 15) | (0b010 << 12) |      \
 143             (0b1110111 << 0) |                     \
 144             (rd << 7) | (rs2 << 20))
 145/* Vector AES final-round decryption. */
 146#define VAESDF_VS(rd, rs2)                          \
 147    ASM_WORD((0b101001 << 26) | (0b1 << 25) |       \
 148             (0b00001 << 15) | (0b010 << 12) |      \
 149             (0b1110111 << 0) |                     \
 150             (rd << 7) | (rs2 << 20))
 151
 152/* Set the key and/or IV into the AES object.
 153 *
 154 * Creates the key schedule from the key.
 155 * Uses Vector Cryptographic instructions.
 156 *
 157 * @param [in] aes     AES object.
 158 * @param [in] key     Secret key to use.
 159 * @param [in] keyLen  Length of key in bytes.
 160 * @param [in] iv      Initialization Vector (IV) to use. May be NULL.
 161 * @param [in] dir     Direction of crypt: AES_ENCRYPT, AES_DECRYPT.
 162 * @return  0 on success.
 163 * @return  BAD_FUNC_ARG when aes or key is NULL.
 164 * @return  BAD_FUNC_ARG when keyLen/dir is not supported or valid.
 165 */
 166int wc_AesSetKey(Aes* aes, const byte* key, word32 keyLen, const byte* iv,
 167    int dir)
 168{
 169    int ret = 0;
 170
 171    /* Validate parameters. */
 172    if ((aes == NULL) || (key == NULL)) {
 173        ret = BAD_FUNC_ARG;
 174    }
 175#ifdef WOLFSSL_AES_128
 176    else if ((keyLen == 16) && (dir == AES_ENCRYPTION)) {
 177        __asm__ __volatile__ (
 178            VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
 179            "mv          t0, %[key]\n\t"
 180            VL1RE32_V(REG_V0, REG_T0)
 181            "mv          t0, %[ks]\n\t"
 182            VAESKF1_VI(REG_V1,  REG_V0, 1)
 183            VAESKF1_VI(REG_V2,  REG_V1, 2)
 184            VAESKF1_VI(REG_V3,  REG_V2, 3)
 185            VAESKF1_VI(REG_V4,  REG_V3, 4)
 186            VAESKF1_VI(REG_V5,  REG_V4, 5)
 187            VAESKF1_VI(REG_V6,  REG_V5, 6)
 188            VAESKF1_VI(REG_V7,  REG_V6, 7)
 189            VAESKF1_VI(REG_V8,  REG_V7, 8)
 190            VAESKF1_VI(REG_V9,  REG_V8, 9)
 191            VAESKF1_VI(REG_V10, REG_V9, 10)
 192            VS8R_V(REG_V0, REG_T0)
 193            "add        t0, t0, 128\n\t"
 194            VS2R_V(REG_V8, REG_T0)
 195            "add        t0, t0, 96\n\t"
 196            VS1R_V(REG_V10, REG_T0)
 197            :
 198            : [ks] "r" (aes->key), [key] "r" (key)
 199            : "memory", "t0", "t1", "t2"
 200        );
 201        aes->rounds = 10;
 202    }
 203#ifdef HAVE_AES_DECRYPT
 204    else if ((keyLen == 16) && (dir == AES_DECRYPTION)) {
 205        __asm__ __volatile__ (
 206            VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
 207            "mv          t0, %[key]\n\t"
 208            VL1RE32_V(REG_V10, REG_T0)
 209            "mv          t0, %[ks]\n\t"
 210            VAESKF1_VI(REG_V9, REG_V10, 1)
 211            VAESKF1_VI(REG_V8, REG_V9 , 2)
 212            VAESKF1_VI(REG_V7, REG_V8 , 3)
 213            VAESKF1_VI(REG_V6, REG_V7 , 4)
 214            VAESKF1_VI(REG_V5, REG_V6 , 5)
 215            VAESKF1_VI(REG_V4, REG_V5 , 6)
 216            VAESKF1_VI(REG_V3, REG_V4 , 7)
 217            VAESKF1_VI(REG_V2, REG_V3 , 8)
 218            VAESKF1_VI(REG_V1, REG_V2 , 9)
 219            VAESKF1_VI(REG_V0, REG_V1 , 10)
 220            VS8R_V(REG_V0, REG_T0)
 221            "add        t0, t0, 128\n\t"
 222            VS2R_V(REG_V8, REG_T0)
 223            "add        t0, t0, 96\n\t"
 224            VS1R_V(REG_V10, REG_T0)
 225            :
 226            : [ks] "r" (aes->key), [key] "r" (key)
 227            : "memory", "t0", "t1", "t2"
 228        );
 229        aes->rounds = 10;
 230    }
 231#endif
 232#endif
 233#ifdef WOLFSSL_AES_192
 234
 235/* One round of computing key schedule for AES-192. */
 236#define AES_192_ROUND(d, s, r)                          \
 237            /* Place key[3] in v16[3] */                \
 238            VSLIDEDOWN_VI(REG_V17, s, 3)                \
 239            VSLIDEUP_VI(REG_V16, REG_V17, 3)            \
 240            /* Place key[5] in s[3] */                  \
 241            VSLIDEUP_VI(s, REG_V14, 3)                  \
 242            /* key'[0] = key[0] ^ ks1(key[5]) */        \
 243            /* key'[1] = key[1] ^ key'[0]     */        \
 244            /* key'[2] = key[2] ^ key'[1]     */        \
 245            /* key'[3] = key[5] ^ key'[2]     */        \
 246            VAESKF1_VI(d, s, r)                         \
 247            /* key'[3] = key[3] ^ key[5] ^ key'[2] */   \
 248            VXOR_VV(d, d, REG_V16)                      \
 249            /* key'[3] = key[3] ^ key'[2] */            \
 250            VSLIDEUP_VI(REG_V16, REG_V14, 3)            \
 251            VXOR_VV(d, d, REG_V16)                      \
 252            /* key'[4] = key[4] ^ key'[3] */            \
 253            VSLIDEDOWN_VI(REG_V15, d, 3)                \
 254            VXOR_VV(REG_V13, REG_V13, REG_V15)          \
 255            /* key'[5] = key[5] ^ key'[4] */            \
 256            VXOR_VV(REG_V14, REG_V14, REG_V13)          \
 257
 258/* Store 6 words.
 259 * V13[0] and V14[0] contain last two words. */
 260#define AES_192_STORE(d)                                \
 261            VS1R_V(d, REG_T0)                           \
 262            "addi        t0, t0, 16\n\t"                \
 263            VSLIDEUP_VI(REG_V13, REG_V14, 1)            \
 264            VSETIVLI(REG_X0, 2, 1, 1, 0b010, 0b000)     \
 265            VS1R_V(REG_V13, REG_T0)                     \
 266            "addi        t0, t0, 8\n\t"                 \
 267            VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
 268
 269    else if ((keyLen == 24) && (dir == AES_ENCRYPTION)) {
 270        /* Not supported with specific instructions - make it work anyway! */
 271        __asm__ __volatile__ (
 272            VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
 273            "mv          t0, %[key]\n\t"
 274            VL1RE32_V(REG_V0, REG_T0)
 275            VSETIVLI(REG_X0, 1, 1, 1, 0b010, 0b000)
 276            "addi        t0, t0, 16\n\t"
 277            VL1RE32_V(REG_V13, REG_T0)
 278            "addi        t0, t0, 4\n\t"
 279            VL1RE32_V(REG_V14, REG_T0)
 280            VXOR_VV(REG_V16, REG_V16, REG_V16)
 281            VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
 282
 283            "mv          t0, %[ks]\n\t"
 284
 285            /* Round 0 */
 286            AES_192_STORE(REG_V0)
 287            /* Round 1 */
 288            AES_192_ROUND(REG_V1, REG_V0, 1)
 289            AES_192_STORE(REG_V1)
 290            /* Round 2 */
 291            AES_192_ROUND(REG_V0, REG_V1, 2)
 292            AES_192_STORE(REG_V0)
 293            /* Round 3 */
 294            AES_192_ROUND(REG_V1, REG_V0, 3)
 295            AES_192_STORE(REG_V1)
 296            /* Round 4 */
 297            AES_192_ROUND(REG_V0, REG_V1, 4)
 298            AES_192_STORE(REG_V0)
 299            /* Round 5 */
 300            AES_192_ROUND(REG_V1, REG_V0, 5)
 301            AES_192_STORE(REG_V1)
 302            /* Round 6 */
 303            AES_192_ROUND(REG_V0, REG_V1, 6)
 304            AES_192_STORE(REG_V0)
 305            /* Round 7 */
 306            AES_192_ROUND(REG_V1, REG_V0, 7)
 307            AES_192_STORE(REG_V1)
 308            /* Round 8 */
 309            AES_192_ROUND(REG_V0, REG_V1, 8)
 310            "addi        t0, t0, 32\n\t"
 311            VS1R_V(REG_V0, REG_T0)
 312            /* Only need 52 32-bit words - 13 rounds x 4 32-bit words. */
 313            :
 314            : [ks] "r" (aes->key), [key] "r" (key)
 315            : "memory", "t0"
 316        );
 317        aes->rounds = 12;
 318    }
 319#ifdef HAVE_AES_DECRYPT
 320    else if ((keyLen == 24) && (dir == AES_DECRYPTION)) {
 321        /* Not supported with specific instructions - make it work anyway! */
 322        __asm__ __volatile__ (
 323            VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
 324            "mv          t0, %[key]\n\t"
 325            VL1RE32_V(REG_V0, REG_T0)
 326            VSETIVLI(REG_X0, 1, 1, 1, 0b010, 0b000)
 327            "addi        t0, t0, 16\n\t"
 328            VL1RE32_V(REG_V13, REG_T0)
 329            "addi        t0, t0, 4\n\t"
 330            VL1RE32_V(REG_V14, REG_T0)
 331            VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
 332            VXOR_VV(REG_V16, REG_V16, REG_V16)
 333
 334            "addi        t0, %[ks], 224\n\t"
 335            VS1R_V(REG_V0, REG_T0)
 336
 337            "mv          t0, %[ks]\n\t"
 338
 339            /* Round 0 */
 340            AES_192_STORE(REG_V0)
 341            /* Round 1 */
 342            AES_192_ROUND(REG_V1, REG_V0, 1)
 343            AES_192_STORE(REG_V1)
 344            /* Round 2 */
 345            AES_192_ROUND(REG_V0, REG_V1, 2)
 346            AES_192_STORE(REG_V0)
 347            /* Round 3 */
 348            AES_192_ROUND(REG_V1, REG_V0, 3)
 349            AES_192_STORE(REG_V1)
 350            /* Round 4 */
 351            AES_192_ROUND(REG_V0, REG_V1, 4)
 352            AES_192_STORE(REG_V0)
 353            /* Round 5 */
 354            AES_192_ROUND(REG_V1, REG_V0, 5)
 355            AES_192_STORE(REG_V1)
 356            /* Round 6 */
 357            AES_192_ROUND(REG_V0, REG_V1, 6)
 358            AES_192_STORE(REG_V0)
 359            /* Round 7 */
 360            AES_192_ROUND(REG_V1, REG_V0, 7)
 361            AES_192_STORE(REG_V1)
 362            /* Round 8 */
 363            AES_192_ROUND(REG_V0, REG_V1, 8)
 364            VS1R_V(REG_V0, REG_T0)
 365            /* Only need 52 32-bit words - 13 rounds x 4 32-bit words. */
 366
 367            /* Invert the order of the round keys. */
 368            "mv          t0, %[ks]\n\t"
 369            VL4RE32_V(REG_V0, REG_T0)
 370            "addi        t0, %[ks], 64\n\t"
 371            VL2RE32_V(REG_V4, REG_T0)
 372            "addi        t1, %[ks], 112\n\t"
 373            VL4RE32_V(REG_V8, REG_T1)
 374            "addi        t1, %[ks], 176\n\t"
 375            VL2RE32_V(REG_V12, REG_T1)
 376            VMV_V_V(REG_V21, REG_V0 )
 377            VMV_V_V(REG_V20, REG_V1 )
 378            VMV_V_V(REG_V19, REG_V2 )
 379            VMV_V_V(REG_V18, REG_V3 )
 380            VMV_V_V(REG_V17, REG_V4 )
 381            VMV_V_V(REG_V16, REG_V5 )
 382            VMV_V_V(REG_V5 , REG_V8 )
 383            VMV_V_V(REG_V4 , REG_V9 )
 384            VMV_V_V(REG_V3 , REG_V10)
 385            VMV_V_V(REG_V2 , REG_V11)
 386            VMV_V_V(REG_V1 , REG_V12)
 387            VMV_V_V(REG_V0 , REG_V13)
 388            "mv          t0, %[ks]\n\t"
 389            VS4R_V(REG_V0, REG_T0)
 390            "addi        t0, %[ks], 64\n\t"
 391            VS2R_V(REG_V4, REG_T0)
 392            "addi        t1, %[ks], 112\n\t"
 393            VS4R_V(REG_V16, REG_T1)
 394            "addi        t1, %[ks], 176\n\t"
 395            VS2R_V(REG_V20, REG_T1)
 396            :
 397            : [ks] "r" (aes->key), [key] "r" (key)
 398            : "memory", "t0", "t1"
 399        );
 400        aes->rounds = 12;
 401    }
 402#endif
 403#endif
 404#ifdef WOLFSSL_AES_256
 405    else if ((keyLen == 32) && (dir == AES_ENCRYPTION)) {
 406        __asm__ __volatile__ (
 407            VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
 408            "mv          t0, %[key]\n\t"
 409            VL2RE32_V(REG_V0, REG_T0)
 410            "mv          t0, %[ks]\n\t"
 411            VMV_V_V(REG_V14, REG_V0)
 412            VMV_V_V(REG_V13, REG_V1)
 413            VAESKF2_VI(REG_V14, REG_V13, 2)
 414            VMV_V_V(REG_V2, REG_V14)
 415            VAESKF2_VI(REG_V13, REG_V14, 3)
 416            VMV_V_V(REG_V3, REG_V13)
 417            VAESKF2_VI(REG_V14, REG_V13, 4)
 418            VMV_V_V(REG_V4, REG_V14)
 419            VAESKF2_VI(REG_V13, REG_V14, 5)
 420            VMV_V_V(REG_V5, REG_V13)
 421            VAESKF2_VI(REG_V14, REG_V13, 6)
 422            VMV_V_V(REG_V6, REG_V14)
 423            VAESKF2_VI(REG_V13, REG_V14, 7)
 424            VMV_V_V(REG_V7, REG_V13)
 425            VAESKF2_VI(REG_V14, REG_V13, 8)
 426            VMV_V_V(REG_V8, REG_V14)
 427            VAESKF2_VI(REG_V13, REG_V14, 9)
 428            VMV_V_V(REG_V9, REG_V13)
 429            VAESKF2_VI(REG_V14, REG_V13, 10)
 430            VMV_V_V(REG_V10, REG_V14)
 431            VAESKF2_VI(REG_V13, REG_V14, 11)
 432            VMV_V_V(REG_V11, REG_V13)
 433            VAESKF2_VI(REG_V14, REG_V13, 12)
 434            VMV_V_V(REG_V12, REG_V14)
 435            VAESKF2_VI(REG_V13, REG_V14, 13)
 436            VAESKF2_VI(REG_V14, REG_V13, 14)
 437            VS8R_V(REG_V0, REG_T0)
 438            "add        t0, t0, 128\n\t"
 439            VSR_V(REG_V8, REG_T0, 4)
 440            "add        t0, t0, 64\n\t"
 441            VSR_V(REG_V12, REG_T0, 2)
 442            "add        t0, t0, 32\n\t"
 443            VSR_V(REG_V14, REG_T0, 1)
 444            :
 445            : [ks] "r" (aes->key), [key] "r" (key)
 446            : "memory", "t0", "t1", "t2"
 447        );
 448        aes->rounds = 14;
 449    }
 450#ifdef HAVE_AES_DECRYPT
 451    else if ((keyLen == 32) && (dir == AES_DECRYPTION)) {
 452        __asm__ __volatile__ (
 453            VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
 454            "mv          t0, %[key]\n\t"
 455            VL2RE32_V(REG_V0, REG_T0)
 456            VMV_V_V(REG_V13, REG_V1)
 457            VMV_V_V(REG_V14, REG_V0)
 458            "mv          t0, %[ks]\n\t"
 459            VAESKF2_VI(REG_V0, REG_V1, 2)
 460            VMV_V_V(REG_V12, REG_V0)
 461            VAESKF2_VI(REG_V1, REG_V0, 3)
 462            VMV_V_V(REG_V11, REG_V1)
 463            VAESKF2_VI(REG_V0, REG_V1, 4)
 464            VMV_V_V(REG_V10, REG_V0)
 465            VAESKF2_VI(REG_V1, REG_V0, 5)
 466            VMV_V_V(REG_V9 , REG_V1)
 467            VAESKF2_VI(REG_V0, REG_V1, 6)
 468            VMV_V_V(REG_V8 , REG_V0)
 469            VAESKF2_VI(REG_V1, REG_V0, 7)
 470            VMV_V_V(REG_V7 , REG_V1)
 471            VAESKF2_VI(REG_V0, REG_V1, 8)
 472            VMV_V_V(REG_V6 , REG_V0)
 473            VAESKF2_VI(REG_V1, REG_V0, 9)
 474            VMV_V_V(REG_V5 , REG_V1)
 475            VAESKF2_VI(REG_V0, REG_V1, 10)
 476            VMV_V_V(REG_V4 , REG_V0)
 477            VAESKF2_VI(REG_V1, REG_V0, 11)
 478            VMV_V_V(REG_V3 , REG_V1)
 479            VAESKF2_VI(REG_V0, REG_V1, 12)
 480            VMV_V_V(REG_V2 , REG_V0)
 481            VAESKF2_VI(REG_V1, REG_V0, 13)
 482            VAESKF2_VI(REG_V0, REG_V1, 14)
 483            VS8R_V(REG_V0, REG_T0)
 484            "add        t0, t0, 128\n\t"
 485            VSR_V(REG_V8, REG_T0, 4)
 486            "add        t0, t0, 64\n\t"
 487            VSR_V(REG_V12, REG_T0, 2)
 488            "add        t0, t0, 32\n\t"
 489            VSR_V(REG_V14, REG_T0, 1)
 490            :
 491            : [ks] "r" (aes->key), [key] "r" (key)
 492            : "memory", "t0", "t1", "t2"
 493        );
 494        aes->rounds = 14;
 495    }
 496#endif
 497#endif
 498    else {
 499        ret = BAD_FUNC_ARG;
 500    }
 501
 502    if (ret == 0) {
 503        /* Set the IV. */
 504        ret = wc_AesSetIV(aes, iv);
 505    }
 506    if (ret == 0) {
 507        /* Finish setting the AES object. */
 508        aes->keylen = keyLen;
 509#if defined(WOLFSSL_AES_COUNTER) || defined(WOLFSSL_AES_CFB) || \
 510    defined(WOLFSSL_AES_OFB) || defined(WOLFSSL_AES_XTS)
 511        aes->left = 0;
 512#endif
 513    }
 514
 515    return ret;
 516}
 517
 518#if defined(WOLFSSL_AES_DIRECT) || defined(HAVE_AESGCM) || defined(HAVE_AESCCM)
 519/* Encrypt a block using AES.
 520 *
 521 * Uses Vector Cryptographic instructions.
 522 *
 523 * @param [in]  aes  AES object.
 524 * @param [in]  in   Block to encrypt.
 525 * @param [out] out  Encrypted block.
 526 */
 527static void wc_AesEncrypt(Aes* aes, const byte* in, byte* out)
 528{
 529    word32* key = aes->key;
 530
 531    __asm__ __volatile__ (
 532        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
 533        /* Load key[0..7]. */
 534        "mv         t0, %[key]\n\t"
 535        VL8RE32_V(REG_V0, REG_T0)
 536        /* Load key[8..9]. */
 537        "addi       t0, t0, 128\n\t"
 538        VL2RE32_V(REG_V8, REG_T0)
 539        /* Check for 11 rounds. */
 540        "li         t4, 11\n\t"
 541        "ble        %[rounds], t4, L_aes_encrypt_loaded\n\t"
 542        /* Load key[10..11]. */
 543        "addi       t0, t0, 32\n\t"
 544        VL2RE32_V(REG_V10, REG_T0)
 545        /* Check for 13 rounds. */
 546        "li         t4, 13\n\t"
 547        "ble        %[rounds], t4, L_aes_encrypt_loaded\n\t"
 548        /* Load key[12..13]. */
 549        "addi       t0, t0, 32\n\t"
 550        VL2RE32_V(REG_V12, REG_T0)
 551      "L_aes_encrypt_loaded:\n\t"
 552        /* Load last round's key */
 553        "addi       t0, %[key], 224\n\t"
 554        VL1RE32_V(REG_V14, REG_T0)
 555
 556        /* Load block. */
 557        "mv         t0, %[in]\n\t"
 558        VL1RE32_V(REG_V15, REG_T0)
 559
 560        /* Encrypt 10 rounds. */
 561        VAESZ_VS(REG_V15, REG_V0)
 562        VAESEM_VS(REG_V15, REG_V1)
 563        VAESEM_VS(REG_V15, REG_V2)
 564        VAESEM_VS(REG_V15, REG_V3)
 565        VAESEM_VS(REG_V15, REG_V4)
 566        VAESEM_VS(REG_V15, REG_V5)
 567        VAESEM_VS(REG_V15, REG_V6)
 568        VAESEM_VS(REG_V15, REG_V7)
 569        VAESEM_VS(REG_V15, REG_V8)
 570        VAESEM_VS(REG_V15, REG_V9)
 571        /* Check for 11 rounds. */
 572        "li         t4, 11\n\t"
 573        "ble        %[rounds], t4, L_aes_encrypt_done\n\t"
 574        VAESEM_VS(REG_V15, REG_V10)
 575        VAESEM_VS(REG_V15, REG_V11)
 576        /* Check for 13 rounds. */
 577        "li         t4, 13\n\t"
 578        "ble        %[rounds], t4, L_aes_encrypt_done\n\t"
 579        VAESEM_VS(REG_V15, REG_V12)
 580        VAESEM_VS(REG_V15, REG_V13)
 581      "L_aes_encrypt_done:\n\t"
 582        /* Last round. */
 583        VAESEF_VS(REG_V15, REG_V14)
 584
 585        /* Store encrypted block. */
 586        "mv         t0, %[out]\n\t"
 587        VS1R_V(REG_V15, REG_T0)
 588
 589        :
 590        : [in] "r" (in), [out] "r" (out), [key] "r" (key),
 591          [rounds] "r" (aes->rounds)
 592        : "memory", "t0", "t1", "t2", "t4"
 593    );
 594}
 595#endif
 596
 597#ifdef HAVE_AES_DECRYPT
 598#ifdef WOLFSSL_AES_DIRECT
 599/* Decrypt a block using AES.
 600 *
 601 * Uses Vector Cryptographic instructions.
 602 *
 603 * @param [in]  aes  AES object.
 604 * @param [in]  in   Block to decrypt.
 605 * @param [out] out  Decrypted block.
 606 */
 607static void wc_AesDecrypt(Aes* aes, const byte* in, byte* out)
 608{
 609    word32* key = aes->key;
 610
 611    __asm__ __volatile__ (
 612        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
 613        /* Load key[0..7]. */
 614        "mv         t0, %[key]\n\t"
 615        VL8RE32_V(REG_V0, REG_T0)
 616        /* Load key[8..9]. */
 617        "addi       t0, t0, 128\n\t"
 618        VL2RE32_V(REG_V8, REG_T0)
 619        /* Check for 11 rounds. */
 620        "li         t4, 11\n\t"
 621        "ble        %[rounds], t4, L_aes_decrypt_loaded\n\t"
 622        /* Load key[10..11]. */
 623        "addi       t0, t0, 32\n\t"
 624        VL2RE32_V(REG_V10, REG_T0)
 625        /* Check for 13 rounds. */
 626        "li         t4, 13\n\t"
 627        "ble        %[rounds], t4, L_aes_decrypt_loaded\n\t"
 628        /* Load key[12..13]. */
 629        "addi       t0, t0, 32\n\t"
 630        VL2RE32_V(REG_V12, REG_T0)
 631      "L_aes_decrypt_loaded:\n\t"
 632        /* Load last round's key */
 633        "addi       t0, %[key], 224\n\t"
 634        VL1RE32_V(REG_V14, REG_T0)
 635
 636        /* Load block. */
 637        "mv         t0, %[in]\n\t"
 638        VL1RE32_V(REG_V15, REG_T0)
 639
 640        /* Decrypt 10 rounds. */
 641        VAESZ_VS(REG_V15, REG_V0)
 642        VAESDM_VS(REG_V15, REG_V1)
 643        VAESDM_VS(REG_V15, REG_V2)
 644        VAESDM_VS(REG_V15, REG_V3)
 645        VAESDM_VS(REG_V15, REG_V4)
 646        VAESDM_VS(REG_V15, REG_V5)
 647        VAESDM_VS(REG_V15, REG_V6)
 648        VAESDM_VS(REG_V15, REG_V7)
 649        VAESDM_VS(REG_V15, REG_V8)
 650        VAESDM_VS(REG_V15, REG_V9)
 651        /* Check for 11 rounds. */
 652        "li         t4, 11\n\t"
 653        "ble        %[rounds], t4, L_aes_decrypt_done\n\t"
 654        VAESDM_VS(REG_V15, REG_V10)
 655        VAESDM_VS(REG_V15, REG_V11)
 656        /* Check for 13 rounds. */
 657        "li         t4, 13\n\t"
 658        "ble        %[rounds], t4, L_aes_decrypt_done\n\t"
 659        VAESDM_VS(REG_V15, REG_V12)
 660        VAESDM_VS(REG_V15, REG_V13)
 661      "L_aes_decrypt_done:\n\t"
 662        /* Last round. */
 663        VAESDF_VS(REG_V15, REG_V14)
 664
 665        /* Store decrypted block. */
 666        "mv         t0, %[out]\n\t"
 667        VS1R_V(REG_V15, REG_T0)
 668
 669        :
 670        : [in] "r" (in), [out] "r" (out), [key] "r" (key),
 671          [rounds] "r" (aes->rounds)
 672        : "memory", "t0", "t1", "t2", "t4"
 673    );
 674}
 675#endif /* WOLFSSL_AES_DIRECT */
 676#endif /* HAVE_AES_DECRYPT */
 677
 678/* AES-CBC */
 679#ifdef HAVE_AES_CBC
 680/* Encrypt blocks of data using AES-CBC.
 681 *
 682 * Uses Vector Cryptographic instructions.
 683 *
 684 * @param [in]  aes  AES object.
 685 * @param [out] out  Encrypted blocks.
 686 * @param [in]  in   Blocks to encrypt.
 687 * @param pin]  sz   Number of bytes to encrypt.
 688 * @return  0 on success.
 689 * @return  BAD_FUNC_ARG when aes, out or in is NULL.
 690 * @return  BAD_LENGTH_E when sz is not a multiple of WC_AES_BLOCK_SIZE.
 691 */
 692int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 693{
 694    int ret = 0;
 695    word32 blocks = sz / WC_AES_BLOCK_SIZE;
 696
 697    /* Validate parameters. */
 698    if ((aes == NULL) || (out == NULL) || (in == NULL)) {
 699        ret = BAD_FUNC_ARG;
 700    }
 701#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 702    /* Ensure a multiple of blocks is to be encrypted.  */
 703    if ((ret == 0) && (sz % WC_AES_BLOCK_SIZE)) {
 704        ret = BAD_LENGTH_E;
 705    }
 706#endif
 707
 708    if ((ret == 0) && (sz > 0)) {
 709        switch (aes->rounds) {
 710#ifdef WOLFSSL_AES_128
 711        case 10:
 712            __asm__ __volatile__ (
 713                VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
 714
 715                /* Load key[0..7]. */
 716                "mv         t0, %[key]\n\t"
 717                VL8RE32_V(REG_V0, REG_T0)
 718                /* Load key[8..9]. */
 719                "addi       t0, t0, 128\n\t"
 720                VL2RE32_V(REG_V8, REG_T0)
 721                /* Load last round's key */
 722                "addi       t0, %[key], 224\n\t"
 723                VL1RE32_V(REG_V10, REG_T0)
 724                /* Load the IV. */
 725                "mv         t0, %[reg]\n\t"
 726                VL1RE32_V(REG_V11, REG_T0)
 727
 728              "L_aes_cbc_128_encrypt_block_loop:\n\t"
 729                /* Load input. */
 730                "mv         t0, %[in]\n\t"
 731                VL1RE32_V(REG_V15, REG_T0)
 732                VXOR_VV(REG_V15, REG_V15, REG_V11)
 733
 734                VAESZ_VS(REG_V15, REG_V0)
 735                VAESEM_VS(REG_V15, REG_V1)
 736                VAESEM_VS(REG_V15, REG_V2)
 737                VAESEM_VS(REG_V15, REG_V3)
 738                VAESEM_VS(REG_V15, REG_V4)
 739                VAESEM_VS(REG_V15, REG_V5)
 740                VAESEM_VS(REG_V15, REG_V6)
 741                VAESEM_VS(REG_V15, REG_V7)
 742                VAESEM_VS(REG_V15, REG_V8)
 743                VAESEM_VS(REG_V15, REG_V9)
 744                VAESEF_VS(REG_V15, REG_V10)
 745
 746                "mv         t0, %[out]\n\t"
 747                VS1R_V(REG_V15, REG_T0)
 748                VMV_V_V(REG_V11, REG_V15)
 749
 750                "addi        %[in], %[in], 16\n\t"
 751                "addi        %[out], %[out], 16\n\t"
 752                /* Loop if more elements to process. */
 753                "addi       %[blocks], %[blocks], -1\n\t"
 754                "bnez       %[blocks], L_aes_cbc_128_encrypt_block_loop\n\t"
 755
 756                "mv         t0, %[reg]\n\t"
 757                VS1R_V(REG_V11, REG_T0)
 758                : [blocks] "+r" (blocks)
 759                : [in] "r" (in), [out] "r" (out), [key] "r" (aes->key),
 760                  [reg] "r" (aes->reg)
 761                : "memory", "t0", "t1", "t2", "t4"
 762            );
 763            break;
 764#endif
 765#ifdef WOLFSSL_AES_192
 766        case 12:
 767            __asm__ __volatile__ (
 768                VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
 769
 770                /* Load key[0..7]. */
 771                "mv         t0, %[key]\n\t"
 772                VL8RE32_V(REG_V0, REG_T0)
 773                /* Load key[8..11]. */
 774                "addi       t0, t0, 128\n\t"
 775                VL4RE32_V(REG_V8, REG_T0)
 776                /* Load last round's key */
 777                "addi       t0, %[key], 224\n\t"
 778                VL1RE32_V(REG_V12, REG_T0)
 779                /* Load the IV. */
 780                "mv         t0, %[reg]\n\t"
 781                VL1RE32_V(REG_V13, REG_T0)
 782
 783              "L_aes_cbc_192_encrypt_block_loop:\n\t"
 784                /* Load input. */
 785                "mv         t0, %[in]\n\t"
 786                VL1RE32_V(REG_V15, REG_T0)
 787                VXOR_VV(REG_V15, REG_V15, REG_V13)
 788
 789                VAESZ_VS(REG_V15, REG_V0)
 790                VAESEM_VS(REG_V15, REG_V1)
 791                VAESEM_VS(REG_V15, REG_V2)
 792                VAESEM_VS(REG_V15, REG_V3)
 793                VAESEM_VS(REG_V15, REG_V4)
 794                VAESEM_VS(REG_V15, REG_V5)
 795                VAESEM_VS(REG_V15, REG_V6)
 796                VAESEM_VS(REG_V15, REG_V7)
 797                VAESEM_VS(REG_V15, REG_V8)
 798                VAESEM_VS(REG_V15, REG_V9)
 799                VAESEM_VS(REG_V15, REG_V10)
 800                VAESEM_VS(REG_V15, REG_V11)
 801                VAESEF_VS(REG_V15, REG_V12)
 802
 803                "mv         t0, %[out]\n\t"
 804                VS1R_V(REG_V15, REG_T0)
 805                VMV_V_V(REG_V13, REG_V15)
 806
 807                "addi        %[in], %[in], 16\n\t"
 808                "addi        %[out], %[out], 16\n\t"
 809                /* Loop if more elements to process. */
 810                "addi       %[blocks], %[blocks], -1\n\t"
 811                "bnez       %[blocks], L_aes_cbc_192_encrypt_block_loop\n\t"
 812
 813                "mv         t0, %[reg]\n\t"
 814                VS1R_V(REG_V13, REG_T0)
 815                : [blocks] "+r" (blocks)
 816                : [in] "r" (in), [out] "r" (out), [key] "r" (aes->key),
 817                  [reg] "r" (aes->reg)
 818                : "memory", "t0", "t1", "t2", "t4"
 819            );
 820            break;
 821#endif
 822#ifdef WOLFSSL_AES_256
 823        case 14:
 824            __asm__ __volatile__ (
 825                VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
 826
 827                /* Load key[0..7]. */
 828                "mv         t0, %[key]\n\t"
 829                VL8RE32_V(REG_V0, REG_T0)
 830                /* Load key[8..11]. */
 831                "addi       t0, t0, 128\n\t"
 832                VL4RE32_V(REG_V8, REG_T0)
 833                /* Load key[12..13]. */
 834                "addi       t0, t0, 64\n\t"
 835                VL2RE32_V(REG_V12, REG_T0)
 836                /* Load last round's key */
 837                "addi       t0, %[key], 224\n\t"
 838                VL1RE32_V(REG_V14, REG_T0)
 839                /* Load the IV. */
 840                "mv         t0, %[reg]\n\t"
 841                VL1RE32_V(REG_V16, REG_T0)
 842
 843              "L_aes_cbc_256_encrypt_block_loop:\n\t"
 844                /* Load input. */
 845                "mv         t0, %[in]\n\t"
 846                VL1RE32_V(REG_V15, REG_T0)
 847                VXOR_VV(REG_V15, REG_V15, REG_V16)
 848
 849                VAESZ_VS(REG_V15, REG_V0)
 850                VAESEM_VS(REG_V15, REG_V1)
 851                VAESEM_VS(REG_V15, REG_V2)
 852                VAESEM_VS(REG_V15, REG_V3)
 853                VAESEM_VS(REG_V15, REG_V4)
 854                VAESEM_VS(REG_V15, REG_V5)
 855                VAESEM_VS(REG_V15, REG_V6)
 856                VAESEM_VS(REG_V15, REG_V7)
 857                VAESEM_VS(REG_V15, REG_V8)
 858                VAESEM_VS(REG_V15, REG_V9)
 859                VAESEM_VS(REG_V15, REG_V10)
 860                VAESEM_VS(REG_V15, REG_V11)
 861                VAESEM_VS(REG_V15, REG_V12)
 862                VAESEM_VS(REG_V15, REG_V13)
 863                VAESEF_VS(REG_V15, REG_V14)
 864
 865                "mv         t0, %[out]\n\t"
 866                VS1R_V(REG_V15, REG_T0)
 867                VMV_V_V(REG_V16, REG_V15)
 868
 869                "addi        %[in], %[in], 16\n\t"
 870                "addi        %[out], %[out], 16\n\t"
 871                /* Loop if more elements to process. */
 872                "addi       %[blocks], %[blocks], -1\n\t"
 873                "bnez       %[blocks], L_aes_cbc_256_encrypt_block_loop\n\t"
 874
 875                "mv         t0, %[reg]\n\t"
 876                VS1R_V(REG_V16, REG_T0)
 877                : [blocks] "+r" (blocks)
 878                : [in] "r" (in), [out] "r" (out), [key] "r" (aes->key),
 879                  [reg] "r" (aes->reg)
 880                : "memory", "t0", "t1", "t2", "t4"
 881            );
 882            break;
 883#endif
 884        }
 885    }
 886
 887    return ret;
 888}
 889
 890#ifdef HAVE_AES_DECRYPT
 891/* Decrypt blocks of data using AES-CBC.
 892 *
 893 * Uses Vector Cryptographic instructions.
 894 *
 895 * @param [in]  aes  AES object.
 896 * @param [out] out  Decrypted blocks.
 897 * @param [in]  in   Blocks to decrypt.
 898 * @param pin]  sz   Number of bytes to decrypt.
 899 * @return  0 on success.
 900 * @return  BAD_FUNC_ARG when aes, out or in is NULL.
 901 * @return  BAD_FUNC_ARG when sz is not a multiple of WC_AES_BLOCK_SIZE.
 902 * @return  BAD_LENGTH_E when sz is not a multiple of WC_AES_BLOCK_SIZE.
 903 */
 904int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 905{
 906    int ret = 0;
 907    word32 blocks = sz / WC_AES_BLOCK_SIZE;
 908
 909    /* Validate parameters. */
 910    if ((aes == NULL) || (out == NULL) || (in == NULL)) {
 911        ret = BAD_FUNC_ARG;
 912    }
 913    /* Ensure a multiple of blocks is being decrypted.  */
 914    if ((ret == 0) && (sz % WC_AES_BLOCK_SIZE)) {
 915#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 916        ret = BAD_LENGTH_E;
 917#else
 918        ret = BAD_FUNC_ARG;
 919#endif
 920    }
 921
 922    if ((ret == 0) && (sz > 0)) {
 923        switch (aes->rounds) {
 924#ifdef WOLFSSL_AES_128
 925        case 10:
 926            __asm__ __volatile__ (
 927                VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
 928
 929                /* Load key[0..7]. */
 930                "mv         t0, %[key]\n\t"
 931                VL8RE32_V(REG_V0, REG_T0)
 932                /* Load key[8..9]. */
 933                "addi       t0, t0, 128\n\t"
 934                VL2RE32_V(REG_V8, REG_T0)
 935                /* Load last round's key */
 936                "addi       t0, %[key], 224\n\t"
 937                VL1RE32_V(REG_V10, REG_T0)
 938                /* Load the IV. */
 939                "mv         t0, %[reg]\n\t"
 940                VL1RE32_V(REG_V11, REG_T0)
 941
 942              "L_aes_cbc_128_decrypt_block_loop:\n\t"
 943                /* Load input. */
 944                "mv         t0, %[in]\n\t"
 945                VL1RE32_V(REG_V15, REG_T0)
 946                VMV_V_V(REG_V14, REG_V15)
 947
 948                VAESZ_VS(REG_V15, REG_V0)
 949                VAESDM_VS(REG_V15, REG_V1)
 950                VAESDM_VS(REG_V15, REG_V2)
 951                VAESDM_VS(REG_V15, REG_V3)
 952                VAESDM_VS(REG_V15, REG_V4)
 953                VAESDM_VS(REG_V15, REG_V5)
 954                VAESDM_VS(REG_V15, REG_V6)
 955                VAESDM_VS(REG_V15, REG_V7)
 956                VAESDM_VS(REG_V15, REG_V8)
 957                VAESDM_VS(REG_V15, REG_V9)
 958                VAESDF_VS(REG_V15, REG_V10)
 959                VXOR_VV(REG_V15, REG_V15, REG_V11)
 960
 961                "mv         t0, %[out]\n\t"
 962                VS1R_V(REG_V15, REG_T0)
 963                VMV_V_V(REG_V11, REG_V14)
 964
 965                "addi        %[in], %[in], 16\n\t"
 966                "addi        %[out], %[out], 16\n\t"
 967                /* Loop if more elements to process. */
 968                "addi       %[blocks], %[blocks], -1\n\t"
 969                "bnez       %[blocks], L_aes_cbc_128_decrypt_block_loop\n\t"
 970
 971                "mv         t0, %[reg]\n\t"
 972                VS1R_V(REG_V11, REG_T0)
 973                : [blocks] "+r" (blocks)
 974                : [in] "r" (in), [out] "r" (out), [key] "r" (aes->key),
 975                  [reg] "r" (aes->reg)
 976                : "memory", "t0", "t1", "t2", "t4"
 977            );
 978            break;
 979#endif
 980#ifdef WOLFSSL_AES_192
 981        case 12:
 982            __asm__ __volatile__ (
 983                VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
 984
 985                /* Load key[0..7]. */
 986                "mv         t0, %[key]\n\t"
 987                VL8RE32_V(REG_V0, REG_T0)
 988                /* Load key[8..11]. */
 989                "addi       t0, t0, 128\n\t"
 990                VL4RE32_V(REG_V8, REG_T0)
 991                /* Load last round's key */
 992                "addi       t0, %[key], 224\n\t"
 993                VL1RE32_V(REG_V12, REG_T0)
 994                /* Load the IV. */
 995                "mv         t0, %[reg]\n\t"
 996                VL1RE32_V(REG_V13, REG_T0)
 997
 998              "L_aes_cbc_192_decrypt_block_loop:\n\t"
 999                /* Load input. */
1000                "mv         t0, %[in]\n\t"
1001                VL1RE32_V(REG_V15, REG_T0)
1002                VMV_V_V(REG_V14, REG_V15)
1003
1004                VAESZ_VS(REG_V15, REG_V0)
1005                VAESDM_VS(REG_V15, REG_V1)
1006                VAESDM_VS(REG_V15, REG_V2)
1007                VAESDM_VS(REG_V15, REG_V3)
1008                VAESDM_VS(REG_V15, REG_V4)
1009                VAESDM_VS(REG_V15, REG_V5)
1010                VAESDM_VS(REG_V15, REG_V6)
1011                VAESDM_VS(REG_V15, REG_V7)
1012                VAESDM_VS(REG_V15, REG_V8)
1013                VAESDM_VS(REG_V15, REG_V9)
1014                VAESDM_VS(REG_V15, REG_V10)
1015                VAESDM_VS(REG_V15, REG_V11)
1016                VAESDF_VS(REG_V15, REG_V12)
1017                VXOR_VV(REG_V15, REG_V15, REG_V13)
1018
1019                "mv         t0, %[out]\n\t"
1020                VS1R_V(REG_V15, REG_T0)
1021                VMV_V_V(REG_V13, REG_V14)
1022
1023                "addi        %[in], %[in], 16\n\t"
1024                "addi        %[out], %[out], 16\n\t"
1025                /* Loop if more elements to process. */
1026                "addi       %[blocks], %[blocks], -1\n\t"
1027                "bnez       %[blocks], L_aes_cbc_192_decrypt_block_loop\n\t"
1028
1029                "mv         t0, %[reg]\n\t"
1030                VS1R_V(REG_V13, REG_T0)
1031                : [blocks] "+r" (blocks)
1032                : [in] "r" (in), [out] "r" (out), [key] "r" (aes->key),
1033                  [reg] "r" (aes->reg)
1034                : "memory", "t0", "t1", "t2", "t4"
1035            );
1036            break;
1037#endif
1038#ifdef WOLFSSL_AES_256
1039        case 14:
1040            __asm__ __volatile__ (
1041                VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
1042
1043                /* Load key[0..7]. */
1044                "mv         t0, %[key]\n\t"
1045                VL8RE32_V(REG_V0, REG_T0)
1046                /* Load key[8..11]. */
1047                "addi       t0, t0, 128\n\t"
1048                VL4RE32_V(REG_V8, REG_T0)
1049                /* Load key[12..13]. */
1050                "addi       t0, t0, 64\n\t"
1051                VL2RE32_V(REG_V12, REG_T0)
1052                /* Load last round's key */
1053                "addi       t0, %[key], 224\n\t"
1054                VL1RE32_V(REG_V14, REG_T0)
1055                /* Load the IV. */
1056                "mv         t0, %[reg]\n\t"
1057                VL1RE32_V(REG_V16, REG_T0)
1058
1059              "L_aes_cbc_256_decrypt_block_loop:\n\t"
1060                /* Load input. */
1061                "mv         t0, %[in]\n\t"
1062                VL1RE32_V(REG_V15, REG_T0)
1063                VMV_V_V(REG_V17, REG_V15)
1064
1065                VAESZ_VS(REG_V15, REG_V0)
1066                VAESDM_VS(REG_V15, REG_V1)
1067                VAESDM_VS(REG_V15, REG_V2)
1068                VAESDM_VS(REG_V15, REG_V3)
1069                VAESDM_VS(REG_V15, REG_V4)
1070                VAESDM_VS(REG_V15, REG_V5)
1071                VAESDM_VS(REG_V15, REG_V6)
1072                VAESDM_VS(REG_V15, REG_V7)
1073                VAESDM_VS(REG_V15, REG_V8)
1074                VAESDM_VS(REG_V15, REG_V9)
1075                VAESDM_VS(REG_V15, REG_V10)
1076                VAESDM_VS(REG_V15, REG_V11)
1077                VAESDM_VS(REG_V15, REG_V12)
1078                VAESDM_VS(REG_V15, REG_V13)
1079                VAESDF_VS(REG_V15, REG_V14)
1080                VXOR_VV(REG_V15, REG_V15, REG_V16)
1081
1082                "mv         t0, %[out]\n\t"
1083                VS1R_V(REG_V15, REG_T0)
1084                VMV_V_V(REG_V16, REG_V17)
1085
1086                "addi        %[in], %[in], 16\n\t"
1087                "addi        %[out], %[out], 16\n\t"
1088                /* Loop if more elements to process. */
1089                "addi       %[blocks], %[blocks], -1\n\t"
1090                "bnez       %[blocks], L_aes_cbc_256_decrypt_block_loop\n\t"
1091
1092                "mv         t0, %[reg]\n\t"
1093                VS1R_V(REG_V16, REG_T0)
1094                : [blocks] "+r" (blocks)
1095                : [in] "r" (in), [out] "r" (out), [key] "r" (aes->key),
1096                  [reg] "r" (aes->reg)
1097                : "memory", "t0", "t1", "t2", "t4"
1098            );
1099            break;
1100#endif
1101        }
1102    }
1103
1104    return ret;
1105}
1106#endif /* HAVE_AES_DECRYPT */
1107
1108/* Don't need generic implementation. */
1109#define HAVE_AES_CBC_ENC_DEC
1110
1111#endif /* HAVE_AES_CBC */
1112
1113/* AES-CTR */
1114#ifdef WOLFSSL_AES_COUNTER
1115/* Encrypt blocks using AES-CTR.
1116 *
1117 * Uses Vector Cryptographic instructions.
1118 *
1119 * @param [in]  aes     AES object.
1120 * @param [out] out     Encrypted blocks.
1121 * @param [in]  in      Blocks to encrypt.
1122 * @param [in]  blocks  Number of blocks to encrypt.
1123 */
1124static void wc_aes_ctr_encrypt_asm(Aes* aes, byte* out, const byte* in,
1125    word32 blocks)
1126{
1127    switch(aes->rounds) {
1128#ifdef WOLFSSL_AES_128
1129    case 10:
1130        __asm__ __volatile__ (
1131            VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
1132
1133            /* Load key[0..7]. */
1134            "mv         t0, %[key]\n\t"
1135            VL8RE32_V(REG_V0, REG_T0)
1136            /* Load key[8..9]. */
1137            "addi       t0, t0, 128\n\t"
1138            VL2RE32_V(REG_V8, REG_T0)
1139            /* Load last round's key */
1140            "addi       t0, %[key], 224\n\t"
1141            VL1RE32_V(REG_V10, REG_T0)
1142            /* Load the counter. */
1143            "mv         t0, %[reg]\n\t"
1144            VL1RE32_V(REG_V16, REG_T0)
1145            "li         t2, 1 \n\t"
1146
1147          "L_aes_ctr_128_encrypt_block_loop:\n\t"
1148            VMV_V_V(REG_V15, REG_V16)
1149            VSETIVLI(REG_X0, 2, 0, 0, 0b011, 0b000)
1150            VREV8(REG_V16, REG_V16)
1151            VSLIDEDOWN_VI(REG_V17, REG_V16, 1)
1152            VXOR_VV(REG_V18, REG_V18, REG_V18)
1153            VADD_VI(REG_V17, REG_V17, 1)
1154            VMSEQ_VV(REG_V18, REG_V18, REG_V17)
1155            VSLIDEUP_VI(REG_V16, REG_V17, 1)
1156            VADD_VV(REG_V16, REG_V16, REG_V18)
1157            VREV8(REG_V16, REG_V16)
1158            VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
1159
1160            VAESZ_VS(REG_V15, REG_V0)
1161            VAESEM_VS(REG_V15, REG_V1)
1162            VAESEM_VS(REG_V15, REG_V2)
1163            VAESEM_VS(REG_V15, REG_V3)
1164            VAESEM_VS(REG_V15, REG_V4)
1165            VAESEM_VS(REG_V15, REG_V5)
1166            VAESEM_VS(REG_V15, REG_V6)
1167            VAESEM_VS(REG_V15, REG_V7)
1168            VAESEM_VS(REG_V15, REG_V8)
1169            VAESEM_VS(REG_V15, REG_V9)
1170            VAESEF_VS(REG_V15, REG_V10)
1171
1172            /* Load input. */
1173            "mv         t0, %[in]\n\t"
1174            VL1RE32_V(REG_V17, REG_T0)
1175            VXOR_VV(REG_V15, REG_V15, REG_V17)
1176
1177            "mv         t0, %[out]\n\t"
1178            VS1R_V(REG_V15, REG_T0)
1179
1180            "addi        %[in], %[in], 16\n\t"
1181            "addi        %[out], %[out], 16\n\t"
1182            /* Loop if more elements to process. */
1183            "addi       %[blocks], %[blocks], -1\n\t"
1184            "bnez       %[blocks], L_aes_ctr_128_encrypt_block_loop\n\t"
1185
1186            "mv         t0, %[reg]\n\t"
1187            VS1R_V(REG_V16, REG_T0)
1188            : [blocks] "+r" (blocks)
1189            : [in] "r" (in), [out] "r" (out), [key] "r" (aes->key),
1190              [reg] "r" (aes->reg)
1191            : "memory", "t0", "t1", "t2", "t4"
1192        );
1193        break;
1194#endif
1195#ifdef WOLFSSL_AES_192
1196    case 12:
1197        __asm__ __volatile__ (
1198            VSETIVLI(REG_X0, 4, 0, 0, 0b010, 0b000)
1199
1200            /* Load key[0..7]. */
1201            "mv         t0, %[key]\n\t"
1202            VL8RE32_V(REG_V0, REG_T0)
1203            /* Load key[8..11]. */
1204            "addi       t0, t0, 128\n\t"
1205            VL4RE32_V(REG_V8, REG_T0)
1206            /* Load last round's key */
1207            "addi       t0, %[key], 224\n\t"
1208            VL1RE32_V(REG_V12, REG_T0)
1209            /* Load the counter. */
1210            "mv         t0, %[reg]\n\t"
1211            VL1RE32_V(REG_V16, REG_T0)
1212            "li         t2, 1 \n\t"
1213
1214          "L_aes_ctr_192_encrypt_block_loop:\n\t"
1215            VMV_V_V(REG_V15, REG_V16)
1216            VSETIVLI(REG_X0, 2, 0, 0, 0b011, 0b000)
1217            VREV8(REG_V16, REG_V16)
1218            VSLIDEDOWN_VI(REG_V17, REG_V16, 1)
1219            VXOR_VV(REG_V18, REG_V18, REG_V18)
1220            VADD_VI(REG_V17, REG_V17, 1)
1221            VMSEQ_VV(REG_V18, REG_V18, REG_V17)
1222            VSLIDEUP_VI(REG_V16, REG_V17, 1)
1223            VADD_VV(REG_V16, REG_V16, REG_V18)
1224            VREV8(REG_V16, REG_V16)
1225            VSETIVLI(REG_X0, 4, 0, 0, 0b010, 0b000)
1226
1227            VAESZ_VS(REG_V15, REG_V0)
1228            VAESEM_VS(REG_V15, REG_V1)
1229            VAESEM_VS(REG_V15, REG_V2)
1230            VAESEM_VS(REG_V15, REG_V3)
1231            VAESEM_VS(REG_V15, REG_V4)
1232            VAESEM_VS(REG_V15, REG_V5)
1233            VAESEM_VS(REG_V15, REG_V6)
1234            VAESEM_VS(REG_V15, REG_V7)
1235            VAESEM_VS(REG_V15, REG_V8)
1236            VAESEM_VS(REG_V15, REG_V9)
1237            VAESEM_VS(REG_V15, REG_V10)
1238            VAESEM_VS(REG_V15, REG_V11)
1239            VAESEF_VS(REG_V15, REG_V12)
1240
1241            /* Load input. */
1242            "mv         t0, %[in]\n\t"
1243            VL1RE32_V(REG_V17, REG_T0)
1244            VXOR_VV(REG_V15, REG_V15, REG_V17)
1245
1246            "mv         t0, %[out]\n\t"
1247            VS1R_V(REG_V15, REG_T0)
1248
1249            "addi        %[in], %[in], 16\n\t"
1250            "addi        %[out], %[out], 16\n\t"
1251            /* Loop if more elements to process. */
1252            "addi       %[blocks], %[blocks], -1\n\t"
1253            "bnez       %[blocks], L_aes_ctr_192_encrypt_block_loop\n\t"
1254
1255            "mv         t0, %[reg]\n\t"
1256            VS1R_V(REG_V16, REG_T0)
1257            : [blocks] "+r" (blocks)
1258            : [in] "r" (in), [out] "r" (out), [key] "r" (aes->key),
1259              [reg] "r" (aes->reg)
1260            : "memory", "t0", "t1", "t2", "t4"
1261        );
1262        break;
1263#endif
1264#ifdef WOLFSSL_AES_256
1265    case 14:
1266        __asm__ __volatile__ (
1267            VSETIVLI(REG_X0, 4, 0, 0, 0b010, 0b000)
1268
1269            /* Load key[0..7]. */
1270            "mv         t0, %[key]\n\t"
1271            VL8RE32_V(REG_V0, REG_T0)
1272            /* Load key[8..11]. */
1273            "addi       t0, t0, 128\n\t"
1274            VL4RE32_V(REG_V8, REG_T0)
1275            /* Load key[12..13]. */
1276            "addi       t0, t0, 64\n\t"
1277            VL2RE32_V(REG_V12, REG_T0)
1278            /* Load last round's key */
1279            "addi       t0, %[key], 224\n\t"
1280            VL1RE32_V(REG_V14, REG_T0)
1281            /* Load the counter. */
1282            "mv         t0, %[reg]\n\t"
1283            VL1RE32_V(REG_V16, REG_T0)
1284            "li         t2, 1 \n\t"
1285
1286          "L_aes_ctr_256_encrypt_block_loop:\n\t"
1287            VMV_V_V(REG_V15, REG_V16)
1288            VSETIVLI(REG_X0, 2, 0, 0, 0b011, 0b000)
1289            VREV8(REG_V16, REG_V16)
1290            VSLIDEDOWN_VI(REG_V17, REG_V16, 1)
1291            VXOR_VV(REG_V18, REG_V18, REG_V18)
1292            VADD_VI(REG_V17, REG_V17, 1)
1293            VMSEQ_VV(REG_V18, REG_V18, REG_V17)
1294            VSLIDEUP_VI(REG_V16, REG_V17, 1)
1295            VADD_VV(REG_V16, REG_V16, REG_V18)
1296            VREV8(REG_V16, REG_V16)
1297            VSETIVLI(REG_X0, 4, 0, 0, 0b010, 0b000)
1298
1299            VAESZ_VS(REG_V15, REG_V0)
1300            VAESEM_VS(REG_V15, REG_V1)
1301            VAESEM_VS(REG_V15, REG_V2)
1302            VAESEM_VS(REG_V15, REG_V3)
1303            VAESEM_VS(REG_V15, REG_V4)
1304            VAESEM_VS(REG_V15, REG_V5)
1305            VAESEM_VS(REG_V15, REG_V6)
1306            VAESEM_VS(REG_V15, REG_V7)
1307            VAESEM_VS(REG_V15, REG_V8)
1308            VAESEM_VS(REG_V15, REG_V9)
1309            VAESEM_VS(REG_V15, REG_V10)
1310            VAESEM_VS(REG_V15, REG_V11)
1311            VAESEM_VS(REG_V15, REG_V12)
1312            VAESEM_VS(REG_V15, REG_V13)
1313            VAESEF_VS(REG_V15, REG_V14)
1314
1315            /* Load input. */
1316            "mv         t0, %[in]\n\t"
1317            VL1RE32_V(REG_V17, REG_T0)
1318            VXOR_VV(REG_V15, REG_V15, REG_V17)
1319
1320            "mv         t0, %[out]\n\t"
1321            VS1R_V(REG_V15, REG_T0)
1322
1323            "addi        %[in], %[in], 16\n\t"
1324            "addi        %[out], %[out], 16\n\t"
1325            /* Loop if more elements to process. */
1326            "addi       %[blocks], %[blocks], -1\n\t"
1327            "bnez       %[blocks], L_aes_ctr_256_encrypt_block_loop\n\t"
1328
1329            "mv         t0, %[reg]\n\t"
1330            VS1R_V(REG_V16, REG_T0)
1331            "mv         t0, %[reg]\n\t"
1332            : [blocks] "+r" (blocks)
1333            : [in] "r" (in), [out] "r" (out), [key] "r" (aes->key),
1334              [reg] "r" (aes->reg)
1335            : "memory", "t0", "t1", "t2", "t4"
1336        );
1337        break;
1338#endif
1339    }
1340}
1341
1342/* Encrypt blocks of data using AES-CTR.
1343 *
1344 * Uses Vector Cryptographic instructions.
1345 *
1346 * @param [in]  aes  AES object.
1347 * @param [out] out  Encrypted blocks.
1348 * @param [in]  in   Blocks to encrypt.
1349 * @param [in]  sz   Number of bytes to encrypt.
1350 * @return  0 on success.
1351 * @return  BAD_FUNC_ARG when aes, out or in is NULL.
1352 * @return  BAD_FUNC_ARG when key size in AES object is not supported.
1353 */
1354int wc_AesCtrEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
1355{
1356   int ret = 0;
1357   word32 processed;
1358
1359    /* Validate parameters. */
1360    if ((aes == NULL) || (out == NULL) || (in == NULL)) {
1361        ret = BAD_FUNC_ARG;
1362    }
1363
1364    if (ret == 0) {
1365        /* Check key size is supported. */
1366        switch (aes->rounds) {
1367        #ifdef WOLFSSL_AES_128
1368            case 10: /* AES 128 BLOCK */
1369        #endif /* WOLFSSL_AES_128 */
1370        #ifdef WOLFSSL_AES_192
1371            case 12: /* AES 192 BLOCK */
1372        #endif /* WOLFSSL_AES_192 */
1373        #ifdef WOLFSSL_AES_256
1374            case 14: /* AES 256 BLOCK */
1375        #endif /* WOLFSSL_AES_256 */
1376                break;
1377            default:
1378                WOLFSSL_MSG("Bad AES-CTR round value");
1379                ret = BAD_FUNC_ARG;
1380        }
1381    }
1382
1383    if (ret == 0) {
1384        /* Use up any unused bytes left in aes->tmp */
1385        processed = min(aes->left, sz);
1386        if (processed > 0) {
1387            /* XOR in encrypted counter.  */
1388            xorbufout(out, in, (byte*)aes->tmp + WC_AES_BLOCK_SIZE - aes->left,
1389                processed);
1390            out += processed;
1391            in += processed;
1392            aes->left -= processed;
1393            sz -= processed;
1394        }
1395
1396        /* Do whole blocks of data. */
1397        while (sz >= WC_AES_BLOCK_SIZE) {
1398            word32 blocks = sz / WC_AES_BLOCK_SIZE;
1399
1400            wc_aes_ctr_encrypt_asm(aes, out, in, blocks);
1401
1402            processed = blocks * WC_AES_BLOCK_SIZE;
1403            out += processed;
1404            in  += processed;
1405            sz  -= processed;
1406            aes->left = 0;
1407        }
1408
1409        if (sz > 0) {
1410            /* Encrypt counter and store in aes->tmp.
1411             * Use up aes->tmp to encrypt data less than a block.
1412             */
1413            static const byte zeros[WC_AES_BLOCK_SIZE] = {
1414                0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
1415            };
1416
1417            wc_aes_ctr_encrypt_asm(aes, (byte*)aes->tmp, zeros, 1);
1418            /* XOR in encrypted counter. */
1419            xorbufout(out, in, aes->tmp, sz);
1420            aes->left = WC_AES_BLOCK_SIZE - sz;
1421        }
1422    }
1423
1424    return ret;
1425}
1426
1427/* Don't need generic implementation. */
1428#define HAVE_AES_COUNTER_ENC
1429
1430#endif /* WOLFSSL_AES_COUNTER */
1431
1432#elif defined(WOLFSSL_RISCV_SCALAR_CRYPTO_ASM)
1433
1434/*
1435 * Scalar crypto instruction implementation of base operations.
1436 */
1437
1438/* AES key schedule SBox operation. */
1439#define AES64KS1I(rd, rs1, rnum)                            \
1440    ASM_WORD(0b00110001000000000001000000010011 |           \
1441             (((rd) << 7) | ((rs1) << 15) | (rnum) << 20))
1442/* AES key schedule operation. */
1443#define AES64KS2(rd, rs1, rs2)                              \
1444    ASM_WORD(0b01111110000000000000000000110011 |           \
1445             (((rd) << 7) | ((rs1) << 15) | (rs2) << 20))
1446/* AES inverse MixColums step. */
1447#define AES64IM(rd, rs1)                                    \
1448    ASM_WORD(0b00110000000000000001000000010011 |           \
1449             ((rd) << 7) | ((rs1) << 15))
1450
1451/* Perform computation of one round of key schedule for AES-128 encryption. */
1452#define AES64_128_KS_RND_INS(rnum, o1, o2)                  \
1453            AES64KS1I(REG_T2, REG_T1, rnum)                 \
1454            AES64KS2(REG_T0, REG_T2, REG_T0)                \
1455            AES64KS2(REG_T1, REG_T0, REG_T1)                \
1456            "sd        t0, " #o1 "(%[ks])\n\t"              \
1457            "sd        t1, " #o2 "(%[ks])\n\t"
1458/* Perform computation of one round of key schedule for AES-128 decryption. */
1459#define AES64_128_INV_KS_RND_INS(rnum, o1, o2)              \
1460            AES64KS1I(REG_T2, REG_T1, rnum)                 \
1461            AES64KS2(REG_T0, REG_T2, REG_T0)                \
1462            AES64KS2(REG_T1, REG_T0, REG_T1)                \
1463            AES64IM(REG_T2, REG_T0)                         \
1464            AES64IM(REG_T3, REG_T1)                         \
1465            "sd        t2, " #o1 "(%[ks])\n\t"              \
1466            "sd        t3, " #o2 "(%[ks])\n\t"
1467
1468/* Perform computation of numbered round of key schedule for AES-128 encryption.
1469 */
1470#define AES64_128_KS_RND(rnum)                              \
1471    AES64_128_KS_RND_INS((rnum), ((rnum) + 1) * 16,         \
1472        ((rnum) + 1) * 16 + 8)
1473/* Perform computation of numbered round of key schedule for AES-128 decryption.
1474 */
1475#define AES64_128_INV_KS_RND(rnum, o)                       \
1476    AES64_128_INV_KS_RND_INS((rnum), (o) * 16, (o) * 16 + 8)
1477/* Perform computation of numbered last round of key schedule for AES-128
1478 * decryption. */
1479#define AES64_128_INV_KS_LRND(rnum, o)                      \
1480    AES64_128_KS_RND_INS((rnum), (o) * 16, (o) * 16 + 8)
1481
1482
1483/* Perform computation of one round of key schedule for AES-192 encryption. */
1484#define AES64_192_KS_RND_INS(rnum, o1, o2, o3)              \
1485            AES64KS1I(REG_T3, REG_T2, rnum)                 \
1486            AES64KS2(REG_T0, REG_T3, REG_T0)                \
1487            AES64KS2(REG_T1, REG_T0, REG_T1)                \
1488            AES64KS2(REG_T2, REG_T1, REG_T2)                \
1489            "sd        t0, " #o1 "(%[ks])\n\t"              \
1490            "sd        t1, " #o2 "(%[ks])\n\t"              \
1491            "sd        t2, " #o3 "(%[ks])\n\t"
1492/* Perform computation of one round of key schedule for AES-192 decryption. */
1493#define AES64_192_INV_KS_RND_INS(rnum, o1, o2, o3)          \
1494            AES64KS1I(REG_T3, REG_T2, rnum)                 \
1495            AES64KS2(REG_T0, REG_T3, REG_T0)                \
1496            AES64KS2(REG_T1, REG_T0, REG_T1)                \
1497            AES64KS2(REG_T2, REG_T1, REG_T2)                \
1498            AES64IM(REG_T3, REG_T0)                         \
1499            AES64IM(REG_T4, REG_T1)                         \
1500            AES64IM(REG_T5, REG_T2)                         \
1501            "sd        t3, " #o1 "(%[ks])\n\t"              \
1502            "sd        t4, " #o2 "(%[ks])\n\t"              \
1503            "sd        t5, " #o3 "(%[ks])\n\t"
1504/* Perform computation of last round of key schedule for AES-192 decryption. */
1505#define AES64_192_KS_LRND_INS(rnum, o1, o2)                 \
1506            AES64KS1I(REG_T3, REG_T2, rnum)                 \
1507            AES64KS2(REG_T0, REG_T3, REG_T0)                \
1508            AES64KS2(REG_T1, REG_T0, REG_T1)                \
1509            "sd        t0, " #o1 "(%[ks])\n\t"              \
1510            "sd        t1, " #o2 "(%[ks])\n\t"              \
1511
1512/* Perform computation of numbered round of key schedule for AES-192 encryption.
1513 */
1514#define AES64_192_KS_RND(rnum)                              \
1515    AES64_192_KS_RND_INS((rnum), ((rnum) + 1) * 24,         \
1516        ((rnum) + 1) * 24 + 8, ((rnum) + 1) * 24 + 16)
1517/* Perform computation of numbered round of key schedule for AES-192 decryption.
1518 */
1519#define AES64_192_INV_KS_RND(rnum, o1, o2, o3)              \
1520    AES64_192_INV_KS_RND_INS((rnum), (o1) * 8, (o2) * 8,    \
1521        (o3) * 8)
1522/* Perform computation of numbered last round of key schedule for AES-192
1523 * encryption. */
1524#define AES64_192_KS_LRND(rnum)                             \
1525    AES64_192_KS_LRND_INS((rnum), ((rnum) + 1) * 24,        \
1526        ((rnum) + 1) * 24 + 8)
1527/* Perform computation of numbered last round of key schedule for AES-192
1528 * decryption. */
1529#define AES64_192_INV_KS_LRND(rnum)                         \
1530    AES64_192_KS_LRND_INS((rnum), 0, 8)
1531
1532
1533/* Perform computation of one round of key schedule for AES-256 encryption. */
1534#define AES64_256_KS_RND_INS(rnum, o1, o2, o3, o4)          \
1535            AES64KS1I(REG_T4, REG_T3, rnum)                 \
1536            AES64KS2(REG_T0, REG_T4, REG_T0)                \
1537            AES64KS2(REG_T1, REG_T0, REG_T1)                \
1538            AES64KS1I(REG_T4, REG_T1, 10)                   \
1539            AES64KS2(REG_T2, REG_T4, REG_T2)                \
1540            AES64KS2(REG_T3, REG_T2, REG_T3)                \
1541            "sd        t0, " #o1 "(%[ks])\n\t"              \
1542            "sd        t1, " #o2 "(%[ks])\n\t"              \
1543            "sd        t2, " #o3 "(%[ks])\n\t"              \
1544            "sd        t3, " #o4 "(%[ks])\n\t"
1545/* Perform computation of one round of key schedule for AES-256 decryption. */
1546#define AES64_256_INV_KS_RND_INS(rnum, o1, o2, o3, o4)      \
1547            AES64KS1I(REG_T4, REG_T3, rnum)                 \
1548            AES64KS2(REG_T0, REG_T4, REG_T0)                \
1549            AES64KS2(REG_T1, REG_T0, REG_T1)                \
1550            AES64KS1I(REG_T4, REG_T1, 10)                   \
1551            AES64KS2(REG_T2, REG_T4, REG_T2)                \
1552            AES64KS2(REG_T3, REG_T2, REG_T3)                \
1553            AES64IM(REG_T4, REG_T0)                         \
1554            AES64IM(REG_T5, REG_T1)                         \
1555            "sd        t4, " #o1 "(%[ks])\n\t"              \
1556            "sd        t5, " #o2 "(%[ks])\n\t"              \
1557            AES64IM(REG_T4, REG_T2)                         \
1558            AES64IM(REG_T5, REG_T3)                         \
1559            "sd        t4, " #o3 "(%[ks])\n\t"              \
1560            "sd        t5, " #o4 "(%[ks])\n\t"
1561/* Perform computation of last round of key schedule for AES-256 decryption. */
1562#define AES64_256_KS_LRND_INS(rnum, o1, o2)                 \
1563            AES64KS1I(REG_T4, REG_T3, rnum)                 \
1564            AES64KS2(REG_T0, REG_T4, REG_T0)                \
1565            AES64KS2(REG_T1, REG_T0, REG_T1)                \
1566            "sd        t0, " #o1 "(%[ks])\n\t"              \
1567            "sd        t1, " #o2 "(%[ks])\n\t"              \
1568
1569/* Perform computation of numbered round of key schedule for AES-256 encryption.
1570 */
1571#define AES64_256_KS_RND(rnum)                              \
1572    AES64_256_KS_RND_INS((rnum), ((rnum) + 1) * 32,         \
1573        ((rnum) + 1) * 32 + 8, ((rnum) + 1) * 32 + 16,      \
1574        ((rnum) + 1) * 32 + 24)
1575/* Perform computation of numbered round of key schedule for AES-256 decryption.
1576 */
1577#define AES64_256_INV_KS_RND(rnum, o)                       \
1578    AES64_256_INV_KS_RND_INS((rnum), (o) * 32,              \
1579        (o) * 32 + 8, (o) * 32 - 16, (o) * 32 - 8)
1580/* Perform computation of numbered last round of key schedule for AES-256
1581 * encryption. */
1582#define AES64_256_KS_LRND(rnum)                             \
1583    AES64_256_KS_LRND_INS((rnum), ((rnum) + 1) * 32,        \
1584        ((rnum) + 1) * 32 + 8)
1585/* Perform computation of numbered last round of key schedule for AES-256
1586 * decryption. */
1587#define AES64_256_INV_KS_LRND(rnum)                         \
1588    AES64_256_KS_LRND_INS((rnum), 0, 8)
1589
1590/* Set the key and/or IV into the AES object.
1591 *
1592 * Creates the key schedule from the key.
1593 * Uses Cryptographic instructions.
1594 *
1595 * @param [in] aes     AES object.
1596 * @param [in] key     Secret key to use.
1597 * @param [in] keyLen  Length of key in bytes.
1598 * @param [in] iv      Initialization Vector (IV) to use. May be NULL.
1599 * @param [in] dir     Direction of crypt: AES_ENCRYPT, AES_DECRYPT.
1600 * @return  0 on success.
1601 * @return  BAD_FUNC_ARG when aes or key is NULL.
1602 * @return  BAD_FUNC_ARG when keyLen/dir is not supported or valid.
1603 */
1604int wc_AesSetKey(Aes* aes, const byte* key, word32 keyLen, const byte* iv,
1605    int dir)
1606{
1607    int ret = 0;
1608
1609    /* Validate parameters. */
1610    if ((aes == NULL) || (key == NULL)) {
1611        ret = BAD_FUNC_ARG;
1612    }
1613#ifdef WOLFSSL_AES_128
1614    else if ((keyLen == 16) && (dir == AES_ENCRYPTION)) {
1615        __asm__ __volatile__ (
1616            "ld        t0, 0(%[key])\n\t"
1617            "ld        t1, 8(%[key])\n\t"
1618            "sd        t0, 0(%[ks])\n\t"
1619            "sd        t1, 8(%[ks])\n\t"
1620            AES64_128_KS_RND(0)
1621            AES64_128_KS_RND(1)
1622            AES64_128_KS_RND(2)
1623            AES64_128_KS_RND(3)
1624            AES64_128_KS_RND(4)
1625            AES64_128_KS_RND(5)
1626            AES64_128_KS_RND(6)
1627            AES64_128_KS_RND(7)
1628            AES64_128_KS_RND(8)
1629            AES64_128_KS_RND(9)
1630            "sd        t0, 224(%[ks])\n\t"
1631            "sd        t1, 232(%[ks])\n\t"
1632            :
1633            : [ks] "r" (aes->key), [key] "r" (key)
1634            : "memory", "t0", "t1", "t2"
1635        );
1636        aes->rounds = 10;
1637    }
1638#ifdef HAVE_AES_DECRYPT
1639    else if ((keyLen == 16) && (dir == AES_DECRYPTION)) {
1640        __asm__ __volatile__ (
1641            "ld        t0, 0(%[key])\n\t"
1642            "ld        t1, 8(%[key])\n\t"
1643            "sd        t0, 160(%[ks])\n\t"
1644            "sd        t1, 168(%[ks])\n\t"
1645            AES64_128_INV_KS_RND(0, 9)
1646            AES64_128_INV_KS_RND(1, 8)
1647            AES64_128_INV_KS_RND(2, 7)
1648            AES64_128_INV_KS_RND(3, 6)
1649            AES64_128_INV_KS_RND(4, 5)
1650            AES64_128_INV_KS_RND(5, 4)
1651            AES64_128_INV_KS_RND(6, 3)
1652            AES64_128_INV_KS_RND(7, 2)
1653            AES64_128_INV_KS_RND(8, 1)
1654            AES64_128_INV_KS_LRND(9, 0)
1655            "sd        t4, 224(%[ks])\n\t"
1656            "sd        t5, 232(%[ks])\n\t"
1657            :
1658            : [ks] "r" (aes->key), [key] "r" (key)
1659            : "memory", "t0", "t1", "t2", "t3"
1660        );
1661        aes->rounds = 10;
1662    }
1663#endif
1664#endif
1665#ifdef WOLFSSL_AES_192
1666    else if ((keyLen == 24) && (dir == AES_ENCRYPTION)) {
1667        __asm__ __volatile__ (
1668            "ld        t0,  0(%[key])\n\t"
1669            "ld        t1,  8(%[key])\n\t"
1670            "ld        t2, 16(%[key])\n\t"
1671            "sd        t0,  0(%[ks])\n\t"
1672            "sd        t1,  8(%[ks])\n\t"
1673            "sd        t2, 16(%[ks])\n\t"
1674            AES64_192_KS_RND(0)
1675            AES64_192_KS_RND(1)
1676            AES64_192_KS_RND(2)
1677            AES64_192_KS_RND(3)
1678            AES64_192_KS_RND(4)
1679            AES64_192_KS_RND(5)
1680            AES64_192_KS_RND(6)
1681            AES64_192_KS_LRND(7)
1682            "sd        t0, 224(%[ks])\n\t"
1683            "sd        t1, 232(%[ks])\n\t"
1684            :
1685            : [ks] "r" (aes->key), [key] "r" (key)
1686            : "memory", "t0", "t1", "t2", "t3"
1687        );
1688        aes->rounds = 12;
1689    }
1690#ifdef HAVE_AES_DECRYPT
1691    else if ((keyLen == 24) && (dir == AES_DECRYPTION)) {
1692        __asm__ __volatile__ (
1693            "ld        t0,  0(%[key])\n\t"
1694            "ld        t1,  8(%[key])\n\t"
1695            "ld        t2, 16(%[key])\n\t"
1696            AES64IM(REG_T3, REG_T2)
1697            "sd        t0, 192(%[ks])\n\t"
1698            "sd        t1, 200(%[ks])\n\t"
1699            "sd        t3, 176(%[ks])\n\t"
1700            AES64_192_INV_KS_RND(0, 23, 20, 21)
1701            AES64_192_INV_KS_RND(1, 18, 19, 16)
1702            AES64_192_INV_KS_RND(2, 17, 14, 15)
1703            AES64_192_INV_KS_RND(3, 12, 13, 10)
1704            AES64_192_INV_KS_RND(4, 11,  8,  9)
1705            AES64_192_INV_KS_RND(5,  6,  7,  4)
1706            AES64_192_INV_KS_RND(6,  5,  2,  3)
1707            AES64_192_INV_KS_LRND(7)
1708            "sd        t4, 224(%[ks])\n\t"
1709            "sd        t5, 232(%[ks])\n\t"
1710            :
1711            : [ks] "r" (aes->key), [key] "r" (key)
1712            : "memory", "t0", "t1", "t2", "t3", "t4", "t5"
1713        );
1714        aes->rounds = 12;
1715    }
1716#endif
1717#endif
1718#ifdef WOLFSSL_AES_256
1719    else if ((keyLen == 32) && (dir == AES_ENCRYPTION)) {
1720        __asm__ __volatile__ (
1721            "ld        t0,  0(%[key])\n\t"
1722            "ld        t1,  8(%[key])\n\t"
1723            "ld        t2, 16(%[key])\n\t"
1724            "ld        t3, 24(%[key])\n\t"
1725            "sd        t0,  0(%[ks])\n\t"
1726            "sd        t1,  8(%[ks])\n\t"
1727            "sd        t2, 16(%[ks])\n\t"
1728            "sd        t3, 24(%[ks])\n\t"
1729            AES64_256_KS_RND(0)
1730            AES64_256_KS_RND(1)
1731            AES64_256_KS_RND(2)
1732            AES64_256_KS_RND(3)
1733            AES64_256_KS_RND(4)
1734            AES64_256_KS_RND(5)
1735            AES64_256_KS_LRND(6)
1736            :
1737            : [ks] "r" (aes->key), [key] "r" (key)
1738            : "memory", "t0", "t1", "t2", "t3", "t4"
1739        );
1740        aes->rounds = 14;
1741    }
1742#ifdef HAVE_AES_DECRYPT
1743    else if ((keyLen == 32) && (dir == AES_DECRYPTION)) {
1744        __asm__ __volatile__ (
1745            "ld        t0,  0(%[key])\n\t"
1746            "ld        t1,  8(%[key])\n\t"
1747            "ld        t2, 16(%[key])\n\t"
1748            "ld        t3, 24(%[key])\n\t"
1749            "sd        t0, 224(%[ks])\n\t"
1750            "sd        t1, 232(%[ks])\n\t"
1751            AES64IM(REG_T4, REG_T2)
1752            AES64IM(REG_T5, REG_T3)
1753            "sd        t4, 208(%[ks])\n\t"
1754            "sd        t5, 216(%[ks])\n\t"
1755            AES64_256_INV_KS_RND(0, 6)
1756            AES64_256_INV_KS_RND(1, 5)
1757            AES64_256_INV_KS_RND(2, 4)
1758            AES64_256_INV_KS_RND(3, 3)
1759            AES64_256_INV_KS_RND(4, 2)
1760            AES64_256_INV_KS_RND(5, 1)
1761            AES64_256_INV_KS_LRND(6)
1762            :
1763            : [ks] "r" (aes->key), [key] "r" (key)
1764            : "memory", "t0", "t1", "t2", "t3", "t4", "t5"
1765        );
1766        aes->rounds = 14;
1767    }
1768#endif
1769#endif
1770    else {
1771        ret = BAD_FUNC_ARG;
1772    }
1773
1774    if (ret == 0) {
1775        /* Set the IV. */
1776        ret = wc_AesSetIV(aes, iv);
1777    }
1778    if (ret == 0) {
1779        /* Finish setting the AES object. */
1780        aes->keylen = keyLen;
1781#if defined(WOLFSSL_AES_COUNTER) || defined(WOLFSSL_AES_CFB) || \
1782    defined(WOLFSSL_AES_OFB) || defined(WOLFSSL_AES_XTS)
1783        aes->left = 0;
1784#endif
1785    }
1786
1787    return ret;
1788}
1789
1790
1791/* AES middle round encryption with 64-bit registers. */
1792#define AES64ESM(rd, rs1, rs2)                      \
1793    ASM_WORD(0b00110110000000000000000000110011 |   \
1794             (rd << 7) | (rs1 << 15) | (rs2 << 20))
1795/* AES final round encryption with 64-bit registers. */
1796#define AES64ES(rd, rs1, rs2)                       \
1797    ASM_WORD(0b00110010000000000000000000110011 |   \
1798             (rd << 7) | (rs1 << 15) | (rs2 << 20))
1799
1800/* Two rounds of encryption.
1801 * kr01 - offset of first half of key for second round.
1802 * kr02 - offset of second half of key for second round.
1803 * kr03 - offset of first half of key for next round.
1804 * kr04 - offset of second half of key for next round.
1805 */
1806#define AESENC_2_ROUNDS(kro1, kro2, kro3, kro4)     \
1807    "ld          a5, " #kro1 " (%[key])\n\t"        \
1808    "ld          a6, " #kro2 " (%[key])\n\t"        \
1809    AES64ESM(REG_T2, REG_T0, REG_T1)                \
1810    AES64ESM(REG_T3, REG_T1, REG_T0)                \
1811    "xor         t2, t2, a3\n\t"                    \
1812    "xor         t3, t3, a4\n\t"                    \
1813    AES64ESM(REG_T0, REG_T2, REG_T3)                \
1814    AES64ESM(REG_T1, REG_T3, REG_T2)                \
1815    "xor         t0, t0, a5\n\t"                    \
1816    "xor         t1, t1, a6\n\t"                    \
1817    "ld          a3, " #kro3 " (%[key])\n\t"        \
1818    "ld          a4, " #kro4 " (%[key])\n\t"
1819
1820/* Last round of encryption. */
1821#define AESENC_LAST_ROUND()                         \
1822    AES64ES(REG_T2, REG_T0, REG_T1)                 \
1823    AES64ES(REG_T3, REG_T1, REG_T0)                 \
1824    "xor         t2, t2, a3\n\t"                    \
1825    "xor         t3, t3, a4\n\t"
1826
1827/* AES middle round decryption with 64-bit registers. */
1828#define AES64DSM(rd, rs1, rs2)                      \
1829    ASM_WORD(0b00111110000000000000000000110011 |   \
1830             (rd << 7) | (rs1 << 15) | (rs2 << 20))
1831/* AES final round decryption with 64-bit registers. */
1832#define AES64DS(rd, rs1, rs2) \
1833    ASM_WORD(0b00111010000000000000000000110011 | \
1834             (rd << 7) | (rs1 << 15) | (rs2 << 20))
1835
1836/* Two rounds of decryption.
1837 * kr01 - offset of first half of key for second round.
1838 * kr02 - offset of second half of key for second round.
1839 * kr03 - offset of first half of key for next round.
1840 * kr04 - offset of second half of key for next round.
1841 */
1842#define AESDEC_2_ROUNDS(kro1, kro2, kro3, kro4)     \
1843    "ld          a5, " #kro1 " (%[key])\n\t"        \
1844    "ld          a6, " #kro2 " (%[key])\n\t"        \
1845    AES64DSM(REG_T2, REG_T0, REG_T1)                \
1846    AES64DSM(REG_T3, REG_T1, REG_T0)                \
1847    "xor         t2, t2, a3\n\t"                    \
1848    "xor         t3, t3, a4\n\t"                    \
1849    AES64DSM(REG_T0, REG_T2, REG_T3)                \
1850    AES64DSM(REG_T1, REG_T3, REG_T2)                \
1851    "xor         t0, t0, a5\n\t"                    \
1852    "xor         t1, t1, a6\n\t"                    \
1853    "ld          a3, " #kro3 " (%[key])\n\t"        \
1854    "ld          a4, " #kro4 " (%[key])\n\t"
1855
1856/* Last round of decryption. */
1857#define AESDEC_LAST_ROUND()                         \
1858    AES64DS(REG_T2, REG_T0, REG_T1)                 \
1859    AES64DS(REG_T3, REG_T1, REG_T0)                 \
1860    "xor         t2, t2, a3\n\t"                    \
1861    "xor         t3, t3, a4\n\t"                    \
1862
1863#if defined(WOLFSSL_AES_DIRECT) || defined(HAVE_AES_CBC) || \
1864    defined(HAVE_AESGCM) || defined(HAVE_AESCCM)
1865/* Encrypt a block using AES.
1866 *
1867 * @param [in]  aes  AES object.
1868 * @param [in]  in   Block to encrypt.
1869 * @param [out] out  Encrypted block.
1870 */
1871static void wc_AesEncrypt(Aes* aes, const byte* in, byte* out)
1872{
1873    __asm__ __volatile__ (
1874        "ld          t2, 0(%[in])\n\t"
1875        "ld          t3, 8(%[in])\n\t"
1876        "ld          a3, 0(%[key])\n\t"
1877        "ld          a4, 8(%[key])\n\t"
1878        "ld          a5, 16(%[key])\n\t"
1879        "ld          a6, 24(%[key])\n\t"
1880        "xor         t2, t2, a3\n\t"
1881        "xor         t3, t3, a4\n\t"
1882        AES64ESM(REG_T0, REG_T2, REG_T3)
1883        AES64ESM(REG_T1, REG_T3, REG_T2)
1884        "xor         t0, t0, a5\n\t"
1885        "xor         t1, t1, a6\n\t"
1886        "ld          a3, 32(%[key])\n\t"
1887        "ld          a4, 40(%[key])\n\t"
1888        AESENC_2_ROUNDS(48, 56, 64, 72)
1889        AESENC_2_ROUNDS(80, 88, 96, 104)
1890        AESENC_2_ROUNDS(112, 120, 128, 136)
1891        AESENC_2_ROUNDS(144, 152, 160, 168)
1892        "li          t4, 11\n\t"
1893        "ble         %[rounds], t4, L_aes_encrypt_done\n\t"
1894        AESENC_2_ROUNDS(176, 184, 192, 200)
1895        "li          t4, 13\n\t"
1896        "ble         %[rounds], t4, L_aes_encrypt_done\n\t"
1897        AESENC_2_ROUNDS(208, 216, 224, 232)
1898      "L_aes_encrypt_done:\n\t"
1899        AESENC_LAST_ROUND()
1900        "sd          t2, 0(%[out])\n\t"
1901        "sd          t3, 8(%[out])\n\t"
1902        :
1903        : [in] "r" (in), [out] "r" (out), [key] "r" (aes->key),
1904          [rounds] "r" (aes->rounds)
1905        : "memory", "t0", "t1", "t2", "t3", "t4", "a3", "a4", "a5", "a6"
1906    );
1907}
1908#endif
1909
1910#ifdef HAVE_AES_DECRYPT
1911#if defined(WOLFSSL_AES_DIRECT) || defined(HAVE_AES_CBC)
1912/* Decrypt a block using AES.
1913 *
1914 * @param [in]  aes  AES object.
1915 * @param [in]  in   Block to decrypt.
1916 * @param [out] out  Decrypted block.
1917 */
1918static void wc_AesDecrypt(Aes* aes, const byte* in, byte* out)
1919{
1920    __asm__ __volatile__ (
1921        "ld          t2, 0(%[in])\n\t"
1922        "ld          t3, 8(%[in])\n\t"
1923        "ld          a3, 0(%[key])\n\t"
1924        "ld          a4, 8(%[key])\n\t"
1925        "ld          a5, 16(%[key])\n\t"
1926        "ld          a6, 24(%[key])\n\t"
1927        "xor         t2, t2, a3\n\t"
1928        "xor         t3, t3, a4\n\t"
1929        AES64DSM(REG_T0, REG_T2, REG_T3)
1930        AES64DSM(REG_T1, REG_T3, REG_T2)
1931        "xor         t0, t0, a5\n\t"
1932        "xor         t1, t1, a6\n\t"
1933        "ld          a3, 32(%[key])\n\t"
1934        "ld          a4, 40(%[key])\n\t"
1935        AESDEC_2_ROUNDS(48, 56, 64, 72)
1936        AESDEC_2_ROUNDS(80, 88, 96, 104)
1937        AESDEC_2_ROUNDS(112, 120, 128, 136)
1938        AESDEC_2_ROUNDS(144, 152, 160, 168)
1939        "li          t4, 11\n\t"
1940        "ble         %[rounds], t4, L_aes_decrypt_done\n\t"
1941        AESDEC_2_ROUNDS(176, 184, 192, 200)
1942        "li          t4, 13\n\t"
1943        "ble         %[rounds], t4, L_aes_decrypt_done\n\t"
1944        AESDEC_2_ROUNDS(208, 216, 224, 232)
1945      "L_aes_decrypt_done:\n\t"
1946        AESDEC_LAST_ROUND()
1947        "sd          t2, 0(%[out])\n\t"
1948        "sd          t3, 8(%[out])\n\t"
1949        :
1950        : [in] "r" (in), [out] "r" (out), [key] "r" (aes->key),
1951          [rounds] "r" (aes->rounds)
1952        : "memory", "t0", "t1", "t2", "t3", "t4", "a3", "a4", "a5", "a6"
1953    );
1954}
1955#endif
1956#endif /* HAVE_AES_DECRYPT */
1957
1958#else
1959
1960/*
1961 * Standard instructions implementation of base operations.
1962 */
1963
1964/* Load a word with bytes reversed. */
1965#define LOAD_WORD_REV(r, o, p)                      \
1966        "lbu        t4, " #o "(" #p ")\n\t"         \
1967        "lbu        t5, " #o "+1(" #p ")\n\t"       \
1968        "lbu        t6, " #o "+2(" #p ")\n\t"       \
1969        "lbu        " #r ", " #o "+3(" #p ")\n\t"   \
1970        "slli       t4, t4, 24\n\t"                 \
1971        "slli       t5, t5, 16\n\t"                 \
1972        "slli       t6, t6, 8\n\t"                  \
1973        "or         " #r ", " #r ", t4\n\t"         \
1974        "or         " #r ", " #r ", t5\n\t"         \
1975        "or         " #r ", " #r ", t6\n\t"
1976
1977/* Store a word with bytes reversed. */
1978#define STORE_WORD_REV(r, o, p)                     \
1979        "srli       t0, " #r ", 24\n\t"             \
1980        "srli       t1, " #r ", 16\n\t"             \
1981        "srli       t2, " #r ", 8\n\t"              \
1982        "sb         t0, " #o "+0(" #p ")\n\t"       \
1983        "sb         t1, " #o "+1(" #p ")\n\t"       \
1984        "sb         t2, " #o "+2(" #p ")\n\t"       \
1985        "sb         " #r ", " #o "+3(" #p ")\n\t"
1986
1987/* AES encryption table. */
1988static const FLASH_QUALIFIER word32 Te[4][256] = {
1989{
1990    0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU,
1991    0xfff2f20dU, 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U,
1992    0x60303050U, 0x02010103U, 0xce6767a9U, 0x562b2b7dU,
1993    0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, 0xec76769aU,
1994    0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U,
1995    0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU,
1996    0x41adadecU, 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU,
1997    0x239c9cbfU, 0x53a4a4f7U, 0xe4727296U, 0x9bc0c05bU,
1998    0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, 0x4c26266aU,
1999    0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU,
2000    0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U,
2001    0xe2717193U, 0xabd8d873U, 0x62313153U, 0x2a15153fU,
2002    0x0804040cU, 0x95c7c752U, 0x46232365U, 0x9dc3c35eU,
2003    0x30181828U, 0x379696a1U, 0x0a05050fU, 0x2f9a9ab5U,
2004    0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU,
2005    0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU,
2006    0x1209091bU, 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU,
2007    0x361b1b2dU, 0xdc6e6eb2U, 0xb45a5aeeU, 0x5ba0a0fbU,
2008    0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, 0x7db3b3ceU,
2009    0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U,
2010    0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU,
2011    0x40202060U, 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU,
2012    0xd46a6abeU, 0x8dcbcb46U, 0x67bebed9U, 0x7239394bU,
2013    0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, 0x85cfcf4aU,
2014    0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U,
2015    0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U,
2016    0x8a4545cfU, 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U,
2017    0xa05050f0U, 0x783c3c44U, 0x259f9fbaU, 0x4ba8a8e3U,
2018    0xa25151f3U, 0x5da3a3feU, 0x804040c0U, 0x058f8f8aU,
2019    0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U,
2020    0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U,
2021    0x20101030U, 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU,
2022    0x81cdcd4cU, 0x180c0c14U, 0x26131335U, 0xc3ecec2fU,
2023    0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, 0x2e171739U,
2024    0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U,
2025    0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U,
2026    0xc06060a0U, 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU,
2027    0x44222266U, 0x542a2a7eU, 0x3b9090abU, 0x0b888883U,
2028    0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, 0x2814143cU,
2029    0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U,
2030    0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU,
2031    0x924949dbU, 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U,
2032    0x9fc2c25dU, 0xbdd3d36eU, 0x43acacefU, 0xc46262a6U,
2033    0x399191a8U, 0x319595a4U, 0xd3e4e437U, 0xf279798bU,
2034    0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U,
2035    0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U,
2036    0xd86c6cb4U, 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U,
2037    0xca6565afU, 0xf47a7a8eU, 0x47aeaee9U, 0x10080818U,
2038    0x6fbabad5U, 0xf0787888U, 0x4a25256fU, 0x5c2e2e72U,
2039    0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U,
2040    0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U,
2041    0x964b4bddU, 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U,
2042    0xe0707090U, 0x7c3e3e42U, 0x71b5b5c4U, 0xcc6666aaU,
2043    0x904848d8U, 0x06030305U, 0xf7f6f601U, 0x1c0e0e12U,
2044    0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U,
2045    0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U,
2046    0xd9e1e138U, 0xebf8f813U, 0x2b9898b3U, 0x22111133U,
2047    0xd26969bbU, 0xa9d9d970U, 0x078e8e89U, 0x339494a7U,
2048    0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, 0xc9e9e920U,
2049    0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU,
2050    0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U,
2051    0x65bfbfdaU, 0xd7e6e631U, 0x844242c6U, 0xd06868b8U,
2052    0x824141c3U, 0x299999b0U, 0x5a2d2d77U, 0x1e0f0f11U,
2053    0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, 0x2c16163aU,
2054},
2055{
2056    0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU,
2057    0x0dfff2f2U, 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U,
2058    0x50603030U, 0x03020101U, 0xa9ce6767U, 0x7d562b2bU,
2059    0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, 0x9aec7676U,
2060    0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU,
2061    0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U,
2062    0xec41adadU, 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU,
2063    0xbf239c9cU, 0xf753a4a4U, 0x96e47272U, 0x5b9bc0c0U,
2064    0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, 0x6a4c2626U,
2065    0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU,
2066    0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U,
2067    0x93e27171U, 0x73abd8d8U, 0x53623131U, 0x3f2a1515U,
2068    0x0c080404U, 0x5295c7c7U, 0x65462323U, 0x5e9dc3c3U,
2069    0x28301818U, 0xa1379696U, 0x0f0a0505U, 0xb52f9a9aU,
2070    0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U,
2071    0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U,
2072    0x1b120909U, 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU,
2073    0x2d361b1bU, 0xb2dc6e6eU, 0xeeb45a5aU, 0xfb5ba0a0U,
2074    0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, 0xce7db3b3U,
2075    0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U,
2076    0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU,
2077    0x60402020U, 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU,
2078    0xbed46a6aU, 0x468dcbcbU, 0xd967bebeU, 0x4b723939U,
2079    0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, 0x4a85cfcfU,
2080    0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU,
2081    0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U,
2082    0xcf8a4545U, 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU,
2083    0xf0a05050U, 0x44783c3cU, 0xba259f9fU, 0xe34ba8a8U,
2084    0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, 0x8a058f8fU,
2085    0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U,
2086    0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U,
2087    0x30201010U, 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U,
2088    0x4c81cdcdU, 0x14180c0cU, 0x35261313U, 0x2fc3ececU,
2089    0xe1be5f5fU, 0xa2359797U, 0xcc884444U, 0x392e1717U,
2090    0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU,
2091    0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U,
2092    0xa0c06060U, 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU,
2093    0x66442222U, 0x7e542a2aU, 0xab3b9090U, 0x830b8888U,
2094    0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, 0x3c281414U,
2095    0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU,
2096    0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU,
2097    0xdb924949U, 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU,
2098    0x5d9fc2c2U, 0x6ebdd3d3U, 0xef43acacU, 0xa6c46262U,
2099    0xa8399191U, 0xa4319595U, 0x37d3e4e4U, 0x8bf27979U,
2100    0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU,
2101    0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U,
2102    0xb4d86c6cU, 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU,
2103    0xafca6565U, 0x8ef47a7aU, 0xe947aeaeU, 0x18100808U,
2104    0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, 0x725c2e2eU,
2105    0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U,
2106    0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU,
2107    0xdd964b4bU, 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU,
2108    0x90e07070U, 0x427c3e3eU, 0xc471b5b5U, 0xaacc6666U,
2109    0xd8904848U, 0x05060303U, 0x01f7f6f6U, 0x121c0e0eU,
2110    0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U,
2111    0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU,
2112    0x38d9e1e1U, 0x13ebf8f8U, 0xb32b9898U, 0x33221111U,
2113    0xbbd26969U, 0x70a9d9d9U, 0x89078e8eU, 0xa7339494U,
2114    0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, 0x20c9e9e9U,
2115    0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU,
2116    0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU,
2117    0xda65bfbfU, 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U,
2118    0xc3824141U, 0xb0299999U, 0x775a2d2dU, 0x111e0f0fU,
2119    0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, 0x3a2c1616U,
2120},
2121{
2122    0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU,
2123    0xf20dfff2U, 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U,
2124    0x30506030U, 0x01030201U, 0x67a9ce67U, 0x2b7d562bU,
2125    0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, 0x769aec76U,
2126    0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU,
2127    0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U,
2128    0xadec41adU, 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU,
2129    0x9cbf239cU, 0xa4f753a4U, 0x7296e472U, 0xc05b9bc0U,
2130    0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, 0x266a4c26U,
2131    0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU,
2132    0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U,
2133    0x7193e271U, 0xd873abd8U, 0x31536231U, 0x153f2a15U,
2134    0x040c0804U, 0xc75295c7U, 0x23654623U, 0xc35e9dc3U,
2135    0x18283018U, 0x96a13796U, 0x050f0a05U, 0x9ab52f9aU,
2136    0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U,
2137    0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U,
2138    0x091b1209U, 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU,
2139    0x1b2d361bU, 0x6eb2dc6eU, 0x5aeeb45aU, 0xa0fb5ba0U,
2140    0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, 0xb3ce7db3U,
2141    0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U,
2142    0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU,
2143    0x20604020U, 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU,
2144    0x6abed46aU, 0xcb468dcbU, 0xbed967beU, 0x394b7239U,
2145    0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, 0xcf4a85cfU,
2146    0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU,
2147    0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U,
2148    0x45cf8a45U, 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU,
2149    0x50f0a050U, 0x3c44783cU, 0x9fba259fU, 0xa8e34ba8U,
2150    0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, 0x8f8a058fU,
2151    0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U,
2152    0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U,
2153    0x10302010U, 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U,
2154    0xcd4c81cdU, 0x0c14180cU, 0x13352613U, 0xec2fc3ecU,
2155    0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, 0x17392e17U,
2156    0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU,
2157    0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U,
2158    0x60a0c060U, 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU,
2159    0x22664422U, 0x2a7e542aU, 0x90ab3b90U, 0x88830b88U,
2160    0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, 0x143c2814U,
2161    0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU,
2162    0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU,
2163    0x49db9249U, 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU,
2164    0xc25d9fc2U, 0xd36ebdd3U, 0xacef43acU, 0x62a6c462U,
2165    0x91a83991U, 0x95a43195U, 0xe437d3e4U, 0x798bf279U,
2166    0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU,
2167    0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U,
2168    0x6cb4d86cU, 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU,
2169    0x65afca65U, 0x7a8ef47aU, 0xaee947aeU, 0x08181008U,
2170    0xbad56fbaU, 0x7888f078U, 0x256f4a25U, 0x2e725c2eU,
2171    0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U,
2172    0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU,
2173    0x4bdd964bU, 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU,
2174    0x7090e070U, 0x3e427c3eU, 0xb5c471b5U, 0x66aacc66U,
2175    0x48d89048U, 0x03050603U, 0xf601f7f6U, 0x0e121c0eU,
2176    0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U,
2177    0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU,
2178    0xe138d9e1U, 0xf813ebf8U, 0x98b32b98U, 0x11332211U,
2179    0x69bbd269U, 0xd970a9d9U, 0x8e89078eU, 0x94a73394U,
2180    0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, 0xe920c9e9U,
2181    0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU,
2182    0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU,
2183    0xbfda65bfU, 0xe631d7e6U, 0x42c68442U, 0x68b8d068U,
2184    0x41c38241U, 0x99b02999U, 0x2d775a2dU, 0x0f111e0fU,
2185    0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, 0x163a2c16U,
2186},
2187{
2188    0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U,
2189    0xf2f20dffU, 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U,
2190    0x30305060U, 0x01010302U, 0x6767a9ceU, 0x2b2b7d56U,
2191    0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, 0x76769aecU,
2192    0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU,
2193    0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU,
2194    0xadadec41U, 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U,
2195    0x9c9cbf23U, 0xa4a4f753U, 0x727296e4U, 0xc0c05b9bU,
2196    0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, 0x26266a4cU,
2197    0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U,
2198    0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U,
2199    0x717193e2U, 0xd8d873abU, 0x31315362U, 0x15153f2aU,
2200    0x04040c08U, 0xc7c75295U, 0x23236546U, 0xc3c35e9dU,
2201    0x18182830U, 0x9696a137U, 0x05050f0aU, 0x9a9ab52fU,
2202    0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU,
2203    0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU,
2204    0x09091b12U, 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U,
2205    0x1b1b2d36U, 0x6e6eb2dcU, 0x5a5aeeb4U, 0xa0a0fb5bU,
2206    0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, 0xb3b3ce7dU,
2207    0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U,
2208    0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U,
2209    0x20206040U, 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U,
2210    0x6a6abed4U, 0xcbcb468dU, 0xbebed967U, 0x39394b72U,
2211    0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, 0xcfcf4a85U,
2212    0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU,
2213    0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U,
2214    0x4545cf8aU, 0xf9f910e9U, 0x02020604U, 0x7f7f81feU,
2215    0x5050f0a0U, 0x3c3c4478U, 0x9f9fba25U, 0xa8a8e34bU,
2216    0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, 0x8f8f8a05U,
2217    0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U,
2218    0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U,
2219    0x10103020U, 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU,
2220    0xcdcd4c81U, 0x0c0c1418U, 0x13133526U, 0xecec2fc3U,
2221    0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, 0x1717392eU,
2222    0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU,
2223    0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U,
2224    0x6060a0c0U, 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U,
2225    0x22226644U, 0x2a2a7e54U, 0x9090ab3bU, 0x8888830bU,
2226    0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, 0x14143c28U,
2227    0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU,
2228    0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U,
2229    0x4949db92U, 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U,
2230    0xc2c25d9fU, 0xd3d36ebdU, 0xacacef43U, 0x6262a6c4U,
2231    0x9191a839U, 0x9595a431U, 0xe4e437d3U, 0x79798bf2U,
2232    0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU,
2233    0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U,
2234    0x6c6cb4d8U, 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU,
2235    0x6565afcaU, 0x7a7a8ef4U, 0xaeaee947U, 0x08081810U,
2236    0xbabad56fU, 0x787888f0U, 0x25256f4aU, 0x2e2e725cU,
2237    0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U,
2238    0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU,
2239    0x4b4bdd96U, 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU,
2240    0x707090e0U, 0x3e3e427cU, 0xb5b5c471U, 0x6666aaccU,
2241    0x4848d890U, 0x03030506U, 0xf6f601f7U, 0x0e0e121cU,
2242    0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U,
2243    0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U,
2244    0xe1e138d9U, 0xf8f813ebU, 0x9898b32bU, 0x11113322U,
2245    0x6969bbd2U, 0xd9d970a9U, 0x8e8e8907U, 0x9494a733U,
2246    0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, 0xe9e920c9U,
2247    0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U,
2248    0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU,
2249    0xbfbfda65U, 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U,
2250    0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU,
2251    0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU,
2252}
2253};
2254
2255/* Round constant used in computing key schedule. */
2256static const FLASH_QUALIFIER word32 rcon[] = {
2257    0x01000000, 0x02000000, 0x04000000, 0x08000000,
2258    0x10000000, 0x20000000, 0x40000000, 0x80000000,
2259    0x1B000000, 0x36000000,
2260    /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
2261};
2262
2263#ifdef HAVE_AES_DECRYPT
2264/* AES decryption table. */
2265static const FLASH_QUALIFIER word32 Td[4][256] = {
2266{
2267    0x51f4a750U, 0x7e416553U, 0x1a17a4c3U, 0x3a275e96U,
2268    0x3bab6bcbU, 0x1f9d45f1U, 0xacfa58abU, 0x4be30393U,
2269    0x2030fa55U, 0xad766df6U, 0x88cc7691U, 0xf5024c25U,
2270    0x4fe5d7fcU, 0xc52acbd7U, 0x26354480U, 0xb562a38fU,
2271    0xdeb15a49U, 0x25ba1b67U, 0x45ea0e98U, 0x5dfec0e1U,
2272    0xc32f7502U, 0x814cf012U, 0x8d4697a3U, 0x6bd3f9c6U,
2273    0x038f5fe7U, 0x15929c95U, 0xbf6d7aebU, 0x955259daU,
2274    0xd4be832dU, 0x587421d3U, 0x49e06929U, 0x8ec9c844U,
2275    0x75c2896aU, 0xf48e7978U, 0x99583e6bU, 0x27b971ddU,
2276    0xbee14fb6U, 0xf088ad17U, 0xc920ac66U, 0x7dce3ab4U,
2277    0x63df4a18U, 0xe51a3182U, 0x97513360U, 0x62537f45U,
2278    0xb16477e0U, 0xbb6bae84U, 0xfe81a01cU, 0xf9082b94U,
2279    0x70486858U, 0x8f45fd19U, 0x94de6c87U, 0x527bf8b7U,
2280    0xab73d323U, 0x724b02e2U, 0xe31f8f57U, 0x6655ab2aU,
2281    0xb2eb2807U, 0x2fb5c203U, 0x86c57b9aU, 0xd33708a5U,
2282    0x302887f2U, 0x23bfa5b2U, 0x02036abaU, 0xed16825cU,
2283    0x8acf1c2bU, 0xa779b492U, 0xf307f2f0U, 0x4e69e2a1U,
2284    0x65daf4cdU, 0x0605bed5U, 0xd134621fU, 0xc4a6fe8aU,
2285    0x342e539dU, 0xa2f355a0U, 0x058ae132U, 0xa4f6eb75U,
2286    0x0b83ec39U, 0x4060efaaU, 0x5e719f06U, 0xbd6e1051U,
2287    0x3e218af9U, 0x96dd063dU, 0xdd3e05aeU, 0x4de6bd46U,
2288    0x91548db5U, 0x71c45d05U, 0x0406d46fU, 0x605015ffU,
2289    0x1998fb24U, 0xd6bde997U, 0x894043ccU, 0x67d99e77U,
2290    0xb0e842bdU, 0x07898b88U, 0xe7195b38U, 0x79c8eedbU,
2291    0xa17c0a47U, 0x7c420fe9U, 0xf8841ec9U, 0x00000000U,
2292    0x09808683U, 0x322bed48U, 0x1e1170acU, 0x6c5a724eU,
2293    0xfd0efffbU, 0x0f853856U, 0x3daed51eU, 0x362d3927U,
2294    0x0a0fd964U, 0x685ca621U, 0x9b5b54d1U, 0x24362e3aU,
2295    0x0c0a67b1U, 0x9357e70fU, 0xb4ee96d2U, 0x1b9b919eU,
2296    0x80c0c54fU, 0x61dc20a2U, 0x5a774b69U, 0x1c121a16U,
2297    0xe293ba0aU, 0xc0a02ae5U, 0x3c22e043U, 0x121b171dU,
2298    0x0e090d0bU, 0xf28bc7adU, 0x2db6a8b9U, 0x141ea9c8U,
2299    0x57f11985U, 0xaf75074cU, 0xee99ddbbU, 0xa37f60fdU,
2300    0xf701269fU, 0x5c72f5bcU, 0x44663bc5U, 0x5bfb7e34U,
2301    0x8b432976U, 0xcb23c6dcU, 0xb6edfc68U, 0xb8e4f163U,
2302    0xd731dccaU, 0x42638510U, 0x13972240U, 0x84c61120U,
2303    0x854a247dU, 0xd2bb3df8U, 0xaef93211U, 0xc729a16dU,
2304    0x1d9e2f4bU, 0xdcb230f3U, 0x0d8652ecU, 0x77c1e3d0U,
2305    0x2bb3166cU, 0xa970b999U, 0x119448faU, 0x47e96422U,
2306    0xa8fc8cc4U, 0xa0f03f1aU, 0x567d2cd8U, 0x223390efU,
2307    0x87494ec7U, 0xd938d1c1U, 0x8ccaa2feU, 0x98d40b36U,
2308    0xa6f581cfU, 0xa57ade28U, 0xdab78e26U, 0x3fadbfa4U,
2309    0x2c3a9de4U, 0x5078920dU, 0x6a5fcc9bU, 0x547e4662U,
2310    0xf68d13c2U, 0x90d8b8e8U, 0x2e39f75eU, 0x82c3aff5U,
2311    0x9f5d80beU, 0x69d0937cU, 0x6fd52da9U, 0xcf2512b3U,
2312    0xc8ac993bU, 0x10187da7U, 0xe89c636eU, 0xdb3bbb7bU,
2313    0xcd267809U, 0x6e5918f4U, 0xec9ab701U, 0x834f9aa8U,
2314    0xe6956e65U, 0xaaffe67eU, 0x21bccf08U, 0xef15e8e6U,
2315    0xbae79bd9U, 0x4a6f36ceU, 0xea9f09d4U, 0x29b07cd6U,
2316    0x31a4b2afU, 0x2a3f2331U, 0xc6a59430U, 0x35a266c0U,
2317    0x744ebc37U, 0xfc82caa6U, 0xe090d0b0U, 0x33a7d815U,
2318    0xf104984aU, 0x41ecdaf7U, 0x7fcd500eU, 0x1791f62fU,
2319    0x764dd68dU, 0x43efb04dU, 0xccaa4d54U, 0xe49604dfU,
2320    0x9ed1b5e3U, 0x4c6a881bU, 0xc12c1fb8U, 0x4665517fU,
2321    0x9d5eea04U, 0x018c355dU, 0xfa877473U, 0xfb0b412eU,
2322    0xb3671d5aU, 0x92dbd252U, 0xe9105633U, 0x6dd64713U,
2323    0x9ad7618cU, 0x37a10c7aU, 0x59f8148eU, 0xeb133c89U,
2324    0xcea927eeU, 0xb761c935U, 0xe11ce5edU, 0x7a47b13cU,
2325    0x9cd2df59U, 0x55f2733fU, 0x1814ce79U, 0x73c737bfU,
2326    0x53f7cdeaU, 0x5ffdaa5bU, 0xdf3d6f14U, 0x7844db86U,
2327    0xcaaff381U, 0xb968c43eU, 0x3824342cU, 0xc2a3405fU,
2328    0x161dc372U, 0xbce2250cU, 0x283c498bU, 0xff0d9541U,
2329    0x39a80171U, 0x080cb3deU, 0xd8b4e49cU, 0x6456c190U,
2330    0x7bcb8461U, 0xd532b670U, 0x486c5c74U, 0xd0b85742U,
2331},
2332{
2333    0x5051f4a7U, 0x537e4165U, 0xc31a17a4U, 0x963a275eU,
2334    0xcb3bab6bU, 0xf11f9d45U, 0xabacfa58U, 0x934be303U,
2335    0x552030faU, 0xf6ad766dU, 0x9188cc76U, 0x25f5024cU,
2336    0xfc4fe5d7U, 0xd7c52acbU, 0x80263544U, 0x8fb562a3U,
2337    0x49deb15aU, 0x6725ba1bU, 0x9845ea0eU, 0xe15dfec0U,
2338    0x02c32f75U, 0x12814cf0U, 0xa38d4697U, 0xc66bd3f9U,
2339    0xe7038f5fU, 0x9515929cU, 0xebbf6d7aU, 0xda955259U,
2340    0x2dd4be83U, 0xd3587421U, 0x2949e069U, 0x448ec9c8U,
2341    0x6a75c289U, 0x78f48e79U, 0x6b99583eU, 0xdd27b971U,
2342    0xb6bee14fU, 0x17f088adU, 0x66c920acU, 0xb47dce3aU,
2343    0x1863df4aU, 0x82e51a31U, 0x60975133U, 0x4562537fU,
2344    0xe0b16477U, 0x84bb6baeU, 0x1cfe81a0U, 0x94f9082bU,
2345    0x58704868U, 0x198f45fdU, 0x8794de6cU, 0xb7527bf8U,
2346    0x23ab73d3U, 0xe2724b02U, 0x57e31f8fU, 0x2a6655abU,
2347    0x07b2eb28U, 0x032fb5c2U, 0x9a86c57bU, 0xa5d33708U,
2348    0xf2302887U, 0xb223bfa5U, 0xba02036aU, 0x5ced1682U,
2349    0x2b8acf1cU, 0x92a779b4U, 0xf0f307f2U, 0xa14e69e2U,
2350    0xcd65daf4U, 0xd50605beU, 0x1fd13462U, 0x8ac4a6feU,
2351    0x9d342e53U, 0xa0a2f355U, 0x32058ae1U, 0x75a4f6ebU,
2352    0x390b83ecU, 0xaa4060efU, 0x065e719fU, 0x51bd6e10U,
2353    0xf93e218aU, 0x3d96dd06U, 0xaedd3e05U, 0x464de6bdU,
2354    0xb591548dU, 0x0571c45dU, 0x6f0406d4U, 0xff605015U,
2355    0x241998fbU, 0x97d6bde9U, 0xcc894043U, 0x7767d99eU,
2356    0xbdb0e842U, 0x8807898bU, 0x38e7195bU, 0xdb79c8eeU,
2357    0x47a17c0aU, 0xe97c420fU, 0xc9f8841eU, 0x00000000U,
2358    0x83098086U, 0x48322bedU, 0xac1e1170U, 0x4e6c5a72U,
2359    0xfbfd0effU, 0x560f8538U, 0x1e3daed5U, 0x27362d39U,
2360    0x640a0fd9U, 0x21685ca6U, 0xd19b5b54U, 0x3a24362eU,
2361    0xb10c0a67U, 0x0f9357e7U, 0xd2b4ee96U, 0x9e1b9b91U,
2362    0x4f80c0c5U, 0xa261dc20U, 0x695a774bU, 0x161c121aU,
2363    0x0ae293baU, 0xe5c0a02aU, 0x433c22e0U, 0x1d121b17U,
2364    0x0b0e090dU, 0xadf28bc7U, 0xb92db6a8U, 0xc8141ea9U,
2365    0x8557f119U, 0x4caf7507U, 0xbbee99ddU, 0xfda37f60U,
2366    0x9ff70126U, 0xbc5c72f5U, 0xc544663bU, 0x345bfb7eU,
2367    0x768b4329U, 0xdccb23c6U, 0x68b6edfcU, 0x63b8e4f1U,
2368    0xcad731dcU, 0x10426385U, 0x40139722U, 0x2084c611U,
2369    0x7d854a24U, 0xf8d2bb3dU, 0x11aef932U, 0x6dc729a1U,
2370    0x4b1d9e2fU, 0xf3dcb230U, 0xec0d8652U, 0xd077c1e3U,
2371    0x6c2bb316U, 0x99a970b9U, 0xfa119448U, 0x2247e964U,
2372    0xc4a8fc8cU, 0x1aa0f03fU, 0xd8567d2cU, 0xef223390U,
2373    0xc787494eU, 0xc1d938d1U, 0xfe8ccaa2U, 0x3698d40bU,
2374    0xcfa6f581U, 0x28a57adeU, 0x26dab78eU, 0xa43fadbfU,
2375    0xe42c3a9dU, 0x0d507892U, 0x9b6a5fccU, 0x62547e46U,
2376    0xc2f68d13U, 0xe890d8b8U, 0x5e2e39f7U, 0xf582c3afU,
2377    0xbe9f5d80U, 0x7c69d093U, 0xa96fd52dU, 0xb3cf2512U,
2378    0x3bc8ac99U, 0xa710187dU, 0x6ee89c63U, 0x7bdb3bbbU,
2379    0x09cd2678U, 0xf46e5918U, 0x01ec9ab7U, 0xa8834f9aU,
2380    0x65e6956eU, 0x7eaaffe6U, 0x0821bccfU, 0xe6ef15e8U,
2381    0xd9bae79bU, 0xce4a6f36U, 0xd4ea9f09U, 0xd629b07cU,
2382    0xaf31a4b2U, 0x312a3f23U, 0x30c6a594U, 0xc035a266U,
2383    0x37744ebcU, 0xa6fc82caU, 0xb0e090d0U, 0x1533a7d8U,
2384    0x4af10498U, 0xf741ecdaU, 0x0e7fcd50U, 0x2f1791f6U,
2385    0x8d764dd6U, 0x4d43efb0U, 0x54ccaa4dU, 0xdfe49604U,
2386    0xe39ed1b5U, 0x1b4c6a88U, 0xb8c12c1fU, 0x7f466551U,
2387    0x049d5eeaU, 0x5d018c35U, 0x73fa8774U, 0x2efb0b41U,
2388    0x5ab3671dU, 0x5292dbd2U, 0x33e91056U, 0x136dd647U,
2389    0x8c9ad761U, 0x7a37a10cU, 0x8e59f814U, 0x89eb133cU,
2390    0xeecea927U, 0x35b761c9U, 0xede11ce5U, 0x3c7a47b1U,
2391    0x599cd2dfU, 0x3f55f273U, 0x791814ceU, 0xbf73c737U,
2392    0xea53f7cdU, 0x5b5ffdaaU, 0x14df3d6fU, 0x867844dbU,
2393    0x81caaff3U, 0x3eb968c4U, 0x2c382434U, 0x5fc2a340U,
2394    0x72161dc3U, 0x0cbce225U, 0x8b283c49U, 0x41ff0d95U,
2395    0x7139a801U, 0xde080cb3U, 0x9cd8b4e4U, 0x906456c1U,
2396    0x617bcb84U, 0x70d532b6U, 0x74486c5cU, 0x42d0b857U,
2397},
2398{
2399    0xa75051f4U, 0x65537e41U, 0xa4c31a17U, 0x5e963a27U,
2400    0x6bcb3babU, 0x45f11f9dU, 0x58abacfaU, 0x03934be3U,
2401    0xfa552030U, 0x6df6ad76U, 0x769188ccU, 0x4c25f502U,
2402    0xd7fc4fe5U, 0xcbd7c52aU, 0x44802635U, 0xa38fb562U,
2403    0x5a49deb1U, 0x1b6725baU, 0x0e9845eaU, 0xc0e15dfeU,
2404    0x7502c32fU, 0xf012814cU, 0x97a38d46U, 0xf9c66bd3U,
2405    0x5fe7038fU, 0x9c951592U, 0x7aebbf6dU, 0x59da9552U,
2406    0x832dd4beU, 0x21d35874U, 0x692949e0U, 0xc8448ec9U,
2407    0x896a75c2U, 0x7978f48eU, 0x3e6b9958U, 0x71dd27b9U,
2408    0x4fb6bee1U, 0xad17f088U, 0xac66c920U, 0x3ab47dceU,
2409    0x4a1863dfU, 0x3182e51aU, 0x33609751U, 0x7f456253U,
2410    0x77e0b164U, 0xae84bb6bU, 0xa01cfe81U, 0x2b94f908U,
2411    0x68587048U, 0xfd198f45U, 0x6c8794deU, 0xf8b7527bU,
2412    0xd323ab73U, 0x02e2724bU, 0x8f57e31fU, 0xab2a6655U,
2413    0x2807b2ebU, 0xc2032fb5U, 0x7b9a86c5U, 0x08a5d337U,
2414    0x87f23028U, 0xa5b223bfU, 0x6aba0203U, 0x825ced16U,
2415    0x1c2b8acfU, 0xb492a779U, 0xf2f0f307U, 0xe2a14e69U,
2416    0xf4cd65daU, 0xbed50605U, 0x621fd134U, 0xfe8ac4a6U,
2417    0x539d342eU, 0x55a0a2f3U, 0xe132058aU, 0xeb75a4f6U,
2418    0xec390b83U, 0xefaa4060U, 0x9f065e71U, 0x1051bd6eU,
2419    0x8af93e21U, 0x063d96ddU, 0x05aedd3eU, 0xbd464de6U,
2420    0x8db59154U, 0x5d0571c4U, 0xd46f0406U, 0x15ff6050U,
2421    0xfb241998U, 0xe997d6bdU, 0x43cc8940U, 0x9e7767d9U,
2422    0x42bdb0e8U, 0x8b880789U, 0x5b38e719U, 0xeedb79c8U,
2423    0x0a47a17cU, 0x0fe97c42U, 0x1ec9f884U, 0x00000000U,
2424    0x86830980U, 0xed48322bU, 0x70ac1e11U, 0x724e6c5aU,
2425    0xfffbfd0eU, 0x38560f85U, 0xd51e3daeU, 0x3927362dU,
2426    0xd9640a0fU, 0xa621685cU, 0x54d19b5bU, 0x2e3a2436U,
2427    0x67b10c0aU, 0xe70f9357U, 0x96d2b4eeU, 0x919e1b9bU,
2428    0xc54f80c0U, 0x20a261dcU, 0x4b695a77U, 0x1a161c12U,
2429    0xba0ae293U, 0x2ae5c0a0U, 0xe0433c22U, 0x171d121bU,
2430    0x0d0b0e09U, 0xc7adf28bU, 0xa8b92db6U, 0xa9c8141eU,
2431    0x198557f1U, 0x074caf75U, 0xddbbee99U, 0x60fda37fU,
2432    0x269ff701U, 0xf5bc5c72U, 0x3bc54466U, 0x7e345bfbU,
2433    0x29768b43U, 0xc6dccb23U, 0xfc68b6edU, 0xf163b8e4U,
2434    0xdccad731U, 0x85104263U, 0x22401397U, 0x112084c6U,
2435    0x247d854aU, 0x3df8d2bbU, 0x3211aef9U, 0xa16dc729U,
2436    0x2f4b1d9eU, 0x30f3dcb2U, 0x52ec0d86U, 0xe3d077c1U,
2437    0x166c2bb3U, 0xb999a970U, 0x48fa1194U, 0x642247e9U,
2438    0x8cc4a8fcU, 0x3f1aa0f0U, 0x2cd8567dU, 0x90ef2233U,
2439    0x4ec78749U, 0xd1c1d938U, 0xa2fe8ccaU, 0x0b3698d4U,
2440    0x81cfa6f5U, 0xde28a57aU, 0x8e26dab7U, 0xbfa43fadU,
2441    0x9de42c3aU, 0x920d5078U, 0xcc9b6a5fU, 0x4662547eU,
2442    0x13c2f68dU, 0xb8e890d8U, 0xf75e2e39U, 0xaff582c3U,
2443    0x80be9f5dU, 0x937c69d0U, 0x2da96fd5U, 0x12b3cf25U,
2444    0x993bc8acU, 0x7da71018U, 0x636ee89cU, 0xbb7bdb3bU,
2445    0x7809cd26U, 0x18f46e59U, 0xb701ec9aU, 0x9aa8834fU,
2446    0x6e65e695U, 0xe67eaaffU, 0xcf0821bcU, 0xe8e6ef15U,
2447    0x9bd9bae7U, 0x36ce4a6fU, 0x09d4ea9fU, 0x7cd629b0U,
2448    0xb2af31a4U, 0x23312a3fU, 0x9430c6a5U, 0x66c035a2U,
2449    0xbc37744eU, 0xcaa6fc82U, 0xd0b0e090U, 0xd81533a7U,
2450    0x984af104U, 0xdaf741ecU, 0x500e7fcdU, 0xf62f1791U,
2451    0xd68d764dU, 0xb04d43efU, 0x4d54ccaaU, 0x04dfe496U,
2452    0xb5e39ed1U, 0x881b4c6aU, 0x1fb8c12cU, 0x517f4665U,
2453    0xea049d5eU, 0x355d018cU, 0x7473fa87U, 0x412efb0bU,
2454    0x1d5ab367U, 0xd25292dbU, 0x5633e910U, 0x47136dd6U,
2455    0x618c9ad7U, 0x0c7a37a1U, 0x148e59f8U, 0x3c89eb13U,
2456    0x27eecea9U, 0xc935b761U, 0xe5ede11cU, 0xb13c7a47U,
2457    0xdf599cd2U, 0x733f55f2U, 0xce791814U, 0x37bf73c7U,
2458    0xcdea53f7U, 0xaa5b5ffdU, 0x6f14df3dU, 0xdb867844U,
2459    0xf381caafU, 0xc43eb968U, 0x342c3824U, 0x405fc2a3U,
2460    0xc372161dU, 0x250cbce2U, 0x498b283cU, 0x9541ff0dU,
2461    0x017139a8U, 0xb3de080cU, 0xe49cd8b4U, 0xc1906456U,
2462    0x84617bcbU, 0xb670d532U, 0x5c74486cU, 0x5742d0b8U,
2463},
2464{
2465    0xf4a75051U, 0x4165537eU, 0x17a4c31aU, 0x275e963aU,
2466    0xab6bcb3bU, 0x9d45f11fU, 0xfa58abacU, 0xe303934bU,
2467    0x30fa5520U, 0x766df6adU, 0xcc769188U, 0x024c25f5U,
2468    0xe5d7fc4fU, 0x2acbd7c5U, 0x35448026U, 0x62a38fb5U,
2469    0xb15a49deU, 0xba1b6725U, 0xea0e9845U, 0xfec0e15dU,
2470    0x2f7502c3U, 0x4cf01281U, 0x4697a38dU, 0xd3f9c66bU,
2471    0x8f5fe703U, 0x929c9515U, 0x6d7aebbfU, 0x5259da95U,
2472    0xbe832dd4U, 0x7421d358U, 0xe0692949U, 0xc9c8448eU,
2473    0xc2896a75U, 0x8e7978f4U, 0x583e6b99U, 0xb971dd27U,
2474    0xe14fb6beU, 0x88ad17f0U, 0x20ac66c9U, 0xce3ab47dU,
2475    0xdf4a1863U, 0x1a3182e5U, 0x51336097U, 0x537f4562U,
2476    0x6477e0b1U, 0x6bae84bbU, 0x81a01cfeU, 0x082b94f9U,
2477    0x48685870U, 0x45fd198fU, 0xde6c8794U, 0x7bf8b752U,
2478    0x73d323abU, 0x4b02e272U, 0x1f8f57e3U, 0x55ab2a66U,
2479    0xeb2807b2U, 0xb5c2032fU, 0xc57b9a86U, 0x3708a5d3U,
2480    0x2887f230U, 0xbfa5b223U, 0x036aba02U, 0x16825cedU,
2481    0xcf1c2b8aU, 0x79b492a7U, 0x07f2f0f3U, 0x69e2a14eU,
2482    0xdaf4cd65U, 0x05bed506U, 0x34621fd1U, 0xa6fe8ac4U,
2483    0x2e539d34U, 0xf355a0a2U, 0x8ae13205U, 0xf6eb75a4U,
2484    0x83ec390bU, 0x60efaa40U, 0x719f065eU, 0x6e1051bdU,
2485    0x218af93eU, 0xdd063d96U, 0x3e05aeddU, 0xe6bd464dU,
2486    0x548db591U, 0xc45d0571U, 0x06d46f04U, 0x5015ff60U,
2487    0x98fb2419U, 0xbde997d6U, 0x4043cc89U, 0xd99e7767U,
2488    0xe842bdb0U, 0x898b8807U, 0x195b38e7U, 0xc8eedb79U,
2489    0x7c0a47a1U, 0x420fe97cU, 0x841ec9f8U, 0x00000000U,
2490    0x80868309U, 0x2bed4832U, 0x1170ac1eU, 0x5a724e6cU,
2491    0x0efffbfdU, 0x8538560fU, 0xaed51e3dU, 0x2d392736U,
2492    0x0fd9640aU, 0x5ca62168U, 0x5b54d19bU, 0x362e3a24U,
2493    0x0a67b10cU, 0x57e70f93U, 0xee96d2b4U, 0x9b919e1bU,
2494    0xc0c54f80U, 0xdc20a261U, 0x774b695aU, 0x121a161cU,
2495    0x93ba0ae2U, 0xa02ae5c0U, 0x22e0433cU, 0x1b171d12U,
2496    0x090d0b0eU, 0x8bc7adf2U, 0xb6a8b92dU, 0x1ea9c814U,
2497    0xf1198557U, 0x75074cafU, 0x99ddbbeeU, 0x7f60fda3U,
2498    0x01269ff7U, 0x72f5bc5cU, 0x663bc544U, 0xfb7e345bU,
2499    0x4329768bU, 0x23c6dccbU, 0xedfc68b6U, 0xe4f163b8U,
2500    0x31dccad7U, 0x63851042U, 0x97224013U, 0xc6112084U,
2501    0x4a247d85U, 0xbb3df8d2U, 0xf93211aeU, 0x29a16dc7U,
2502    0x9e2f4b1dU, 0xb230f3dcU, 0x8652ec0dU, 0xc1e3d077U,
2503    0xb3166c2bU, 0x70b999a9U, 0x9448fa11U, 0xe9642247U,
2504    0xfc8cc4a8U, 0xf03f1aa0U, 0x7d2cd856U, 0x3390ef22U,
2505    0x494ec787U, 0x38d1c1d9U, 0xcaa2fe8cU, 0xd40b3698U,
2506    0xf581cfa6U, 0x7ade28a5U, 0xb78e26daU, 0xadbfa43fU,
2507    0x3a9de42cU, 0x78920d50U, 0x5fcc9b6aU, 0x7e466254U,
2508    0x8d13c2f6U, 0xd8b8e890U, 0x39f75e2eU, 0xc3aff582U,
2509    0x5d80be9fU, 0xd0937c69U, 0xd52da96fU, 0x2512b3cfU,
2510    0xac993bc8U, 0x187da710U, 0x9c636ee8U, 0x3bbb7bdbU,
2511    0x267809cdU, 0x5918f46eU, 0x9ab701ecU, 0x4f9aa883U,
2512    0x956e65e6U, 0xffe67eaaU, 0xbccf0821U, 0x15e8e6efU,
2513    0xe79bd9baU, 0x6f36ce4aU, 0x9f09d4eaU, 0xb07cd629U,
2514    0xa4b2af31U, 0x3f23312aU, 0xa59430c6U, 0xa266c035U,
2515    0x4ebc3774U, 0x82caa6fcU, 0x90d0b0e0U, 0xa7d81533U,
2516    0x04984af1U, 0xecdaf741U, 0xcd500e7fU, 0x91f62f17U,
2517    0x4dd68d76U, 0xefb04d43U, 0xaa4d54ccU, 0x9604dfe4U,
2518    0xd1b5e39eU, 0x6a881b4cU, 0x2c1fb8c1U, 0x65517f46U,
2519    0x5eea049dU, 0x8c355d01U, 0x877473faU, 0x0b412efbU,
2520    0x671d5ab3U, 0xdbd25292U, 0x105633e9U, 0xd647136dU,
2521    0xd7618c9aU, 0xa10c7a37U, 0xf8148e59U, 0x133c89ebU,
2522    0xa927eeceU, 0x61c935b7U, 0x1ce5ede1U, 0x47b13c7aU,
2523    0xd2df599cU, 0xf2733f55U, 0x14ce7918U, 0xc737bf73U,
2524    0xf7cdea53U, 0xfdaa5b5fU, 0x3d6f14dfU, 0x44db8678U,
2525    0xaff381caU, 0x68c43eb9U, 0x24342c38U, 0xa3405fc2U,
2526    0x1dc37216U, 0xe2250cbcU, 0x3c498b28U, 0x0d9541ffU,
2527    0xa8017139U, 0x0cb3de08U, 0xb4e49cd8U, 0x56c19064U,
2528    0xcb84617bU, 0x32b670d5U, 0x6c5c7448U, 0xb85742d0U,
2529}
2530};
2531
2532#endif /* HAVE_AES_DECRYPT */
2533
2534/* AES substitute rotated word with round constant application. */
2535#define AES_SUB_ROT_WORD_RCON(s)            \
2536            "andi   t5, " #s ", 0xff\n\t"   \
2537            "slli   t5, t5, 2\n\t"          \
2538            "add    t5, t5, %[te]\n\t"      \
2539            "lw     t5, (t5)\n\t"           \
2540            "li     t6, 0x0000ff00\n\t"     \
2541            "and    t5, t5, t6\n\t"         \
2542            "xor    t0, t0, t5\n\t"         \
2543                                            \
2544            "srli   t5, " #s ", 22\n\t"     \
2545            "andi   t5, t5, 0x3fc\n\t"      \
2546            "add    t5, t5, %[te]\n\t"      \
2547            "addi   t5, t5, 0x400\n\t"      \
2548            "lbu    t5, (t5)\n\t"           \
2549            "xor    t0, t0, t5\n\t"         \
2550                                            \
2551            "srli   t5, " #s ", 14\n\t"     \
2552            "andi   t5, t5, 0x3fc\n\t"      \
2553            "add    t5, t5, %[te]\n\t"      \
2554            "li     t6, 0x800\n\t"          \
2555            "add    t5, t5, t6\n\t"         \
2556            "lw     t5, (t5)\n\t"           \
2557            "li     t6, 0xff000000\n\t"     \
2558            "and    t5, t5, t6\n\t"         \
2559            "xor    t0, t0, t5\n\t"         \
2560                                            \
2561            "srli   t5, " #s ", 6\n\t"      \
2562            "andi   t5, t5, 0x3fc\n\t"      \
2563            "add    t5, t5, %[te]\n\t"      \
2564            "li     t6, 0xc00\n\t"          \
2565            "add    t5, t5, t6\n\t"         \
2566            "lw     t5, (t5)\n\t"           \
2567            "li     t6, 0x00ff0000\n\t"     \
2568            "and    t5, t5, t6\n\t"         \
2569            "xor    t0, t0, t5\n\t"         \
2570                                            \
2571            "add    t5, %[rcon], a5\n\t"    \
2572            "lw     t5, (t5)\n\t"           \
2573            "xor    t0, t0, t5\n\t"
2574
2575/* AES substitute word. */
2576#define AES_SUB_WORD(s)                     \
2577            "srli   t5, " #s ", 6\n\t"      \
2578            "andi   t5, t5, 0x3fc\n\t"      \
2579            "add    t5, t5, %[te]\n\t"      \
2580            "lw     t5, (t5)\n\t"           \
2581            "li     t6, 0x0000ff00\n\t"     \
2582            "and    t5, t5, t6\n\t"         \
2583            "xor    a6, a6, t5\n\t"         \
2584                                            \
2585            "andi   t5, " #s ", 0xff\n\t"   \
2586            "slli   t5, t5, 2\n\t"          \
2587            "add    t5, t5, %[te]\n\t"      \
2588            "addi   t5, t5, 0x400\n\t"      \
2589            "lbu    t5, (t5)\n\t"           \
2590            "xor    a6, a6, t5\n\t"         \
2591                                            \
2592            "srli   t5, " #s ", 22\n\t"     \
2593            "andi   t5, t5, 0x3fc\n\t"      \
2594            "add    t5, t5, %[te]\n\t"      \
2595            "li     t6, 0x800\n\t"          \
2596            "add    t5, t5, t6\n\t"         \
2597            "lw     t5, (t5)\n\t"           \
2598            "li     t6, 0xff000000\n\t"     \
2599            "and    t5, t5, t6\n\t"         \
2600            "xor    a6, a6, t5\n\t"         \
2601                                            \
2602            "srli   t5, " #s ", 14\n\t"     \
2603            "andi   t5, t5, 0x3fc\n\t"      \
2604            "add    t5, t5, %[te]\n\t"      \
2605            "li     t6, 0xc00\n\t"          \
2606            "add    t5, t5, t6\n\t"         \
2607            "lw     t5, (t5)\n\t"           \
2608            "li     t6, 0x00ff0000\n\t"     \
2609            "and    t5, t5, t6\n\t"         \
2610            "xor    a6, a6, t5\n\t"
2611
2612/* Set the AES key and expand.
2613 *
2614 * @param [in]  aes    AES object.
2615 * @param [in]  key    Block to encrypt.
2616 * @param [in]  keySz  Number of bytes in key.
2617 * @param [in]  dir    Direction of crypt: AES_ENCRYPTION or AES_DECRYPTION.
2618 */
2619static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir)
2620{
2621    word32* rk = aes->key;
2622
2623    switch (keySz) {
2624#if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 128 && \
2625        defined(WOLFSSL_AES_128)
2626    case 16:
2627        __asm__ __volatile__ (
2628#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
2629            /* Load 4 32-bit words in reverse byte order. */
2630            LOAD_WORD_REV(t0, 0, %[key])
2631            LOAD_WORD_REV(t1, 4, %[key])
2632            LOAD_WORD_REV(t2, 8, %[key])
2633            LOAD_WORD_REV(t3, 12, %[key])
2634#else
2635            "ld     t1, 0(%[key])\n\t"
2636            "ld     t3, 8(%[key])\n\t"
2637            REV8(REG_T1, REG_T1)
2638            REV8(REG_T3, REG_T3)
2639            "srli   t0, t1, 32\n\t"
2640            "srli   t2, t3, 32\n\t"
2641#endif
2642            /* Store round 0 key. */
2643            "sw     t0,  0(%[rk])\n\t"
2644            "sw     t1,  4(%[rk])\n\t"
2645            "sw     t2,  8(%[rk])\n\t"
2646            "sw     t3, 12(%[rk])\n\t"
2647
2648            "li     a4, 10\n\t"
2649            "mv     a5, x0\n\t"
2650        "L_aes_set_key_c_16_loop:\n\t"
2651            "addi   %[rk], %[rk], 16\n\t"
2652            /* Permute key. */
2653            AES_SUB_ROT_WORD_RCON(t3)
2654            "xor    t1, t1, t0\n\t"
2655            "xor    t2, t2, t1\n\t"
2656            "xor    t3, t3, t2\n\t"
2657            /* Store round key. */
2658            "sw     t0,  0(%[rk])\n\t"
2659            "sw     t1,  4(%[rk])\n\t"
2660            "sw     t2,  8(%[rk])\n\t"
2661            "sw     t3, 12(%[rk])\n\t"
2662
2663            "addi   a4, a4, -1\n\t"
2664            "addi   a5, a5, 4\n\t"
2665            "bnez   a4, L_aes_set_key_c_16_loop\n\t"
2666            : [rk] "+r" (rk)
2667            : [key] "r" (key), [te] "r" (Te), [rcon] "r" (rcon)
2668            : "memory", "t0", "t1", "t2", "t3", "t4", "t5", "t6", "a4", "a5"
2669        );
2670        break;
2671#endif /* 128 */
2672
2673#if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 192 && \
2674        defined(WOLFSSL_AES_192)
2675    case 24:
2676        __asm__ __volatile__ (
2677#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
2678            /* Load 6 32-bit words in reverse byte order. */
2679            LOAD_WORD_REV(t0, 0, %[key])
2680            LOAD_WORD_REV(t1, 4, %[key])
2681            LOAD_WORD_REV(t2, 8, %[key])
2682            LOAD_WORD_REV(t3, 12, %[key])
2683            LOAD_WORD_REV(a6, 16, %[key])
2684            LOAD_WORD_REV(a7, 20, %[key])
2685#else
2686            "ld     t1, 0(%[key])\n\t"
2687            "ld     t3, 8(%[key])\n\t"
2688            "ld     a7, 16(%[key])\n\t"
2689            REV8(REG_T1, REG_T1)
2690            REV8(REG_T3, REG_T3)
2691            REV8(REG_A7, REG_A7)
2692            "srli   t0, t1, 32\n\t"
2693            "srli   t2, t3, 32\n\t"
2694            "srli   a6, a7, 32\n\t"
2695#endif
2696            /* Store round 0 key. */
2697            "sw     t0,  0(%[rk])\n\t"
2698            "sw     t1,  4(%[rk])\n\t"
2699            "sw     t2,  8(%[rk])\n\t"
2700            "sw     t3, 12(%[rk])\n\t"
2701            "sw     a6, 16(%[rk])\n\t"
2702            "sw     a7, 20(%[rk])\n\t"
2703
2704            "li     a4, 8\n\t"
2705            "mv     a5, x0\n\t"
2706        "L_aes_set_key_c_24_loop:\n\t"
2707            "addi   %[rk], %[rk], 24\n\t"
2708            /* Permute key. */
2709            AES_SUB_ROT_WORD_RCON(a7)
2710            "xor    t1, t1, t0\n\t"
2711            "xor    t2, t2, t1\n\t"
2712            "xor    t3, t3, t2\n\t"
2713            "xor    a6, a6, t3\n\t"
2714            "xor    a7, a7, a6\n\t"
2715            /* Store round key. */
2716            "sw     t0,  0(%[rk])\n\t"
2717            "sw     t1,  4(%[rk])\n\t"
2718            "sw     t2,  8(%[rk])\n\t"
2719            "sw     t3, 12(%[rk])\n\t"
2720            "sw     a6, 16(%[rk])\n\t"
2721            "sw     a7, 20(%[rk])\n\t"
2722
2723            "addi   a4, a4, -1\n\t"
2724            "addi   a5, a5, 4\n\t"
2725            "bnez   a4, L_aes_set_key_c_24_loop\n\t"
2726
2727            : [rk] "+r" (rk)
2728            : [key] "r" (key), [te] "r" (Te), [rcon] "r" (rcon)
2729            : "memory", "t0", "t1", "t2", "t3", "t4", "t5", "t6", "a4", "a5",
2730              "a6", "a7"
2731        );
2732        break;
2733#endif /* 192 */
2734
2735#if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 256 && \
2736        defined(WOLFSSL_AES_256)
2737    case 32:
2738        __asm__ __volatile__ (
2739#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
2740            /* Load 8 32-bit words in reverse byte order. */
2741            LOAD_WORD_REV(t0, 0, %[key])
2742            LOAD_WORD_REV(t1, 4, %[key])
2743            LOAD_WORD_REV(t2, 8, %[key])
2744            LOAD_WORD_REV(t3, 12, %[key])
2745            LOAD_WORD_REV(a6, 16, %[key])
2746            LOAD_WORD_REV(a7, 20, %[key])
2747            LOAD_WORD_REV(s1, 24, %[key])
2748            LOAD_WORD_REV(s2, 28, %[key])
2749#else
2750            "ld     t1, 0(%[key])\n\t"
2751            "ld     t3, 8(%[key])\n\t"
2752            "ld     a7, 16(%[key])\n\t"
2753            "ld     s2, 24(%[key])\n\t"
2754            REV8(REG_T1, REG_T1)
2755            REV8(REG_T3, REG_T3)
2756            REV8(REG_A7, REG_A7)
2757            REV8(REG_S2, REG_S2)
2758            "srli   t0, t1, 32\n\t"
2759            "srli   t2, t3, 32\n\t"
2760            "srli   a6, a7, 32\n\t"
2761            "srli   s1, s2, 32\n\t"
2762#endif
2763            /* Store round 0 key. */
2764            "sw     t0,  0(%[rk])\n\t"
2765            "sw     t1,  4(%[rk])\n\t"
2766            "sw     t2,  8(%[rk])\n\t"
2767            "sw     t3, 12(%[rk])\n\t"
2768            "sw     a6, 16(%[rk])\n\t"
2769            "sw     a7, 20(%[rk])\n\t"
2770            "sw     s1, 24(%[rk])\n\t"
2771            "sw     s2, 28(%[rk])\n\t"
2772
2773            "li     a4, 7\n\t"
2774            "mv     a5, x0\n\t"
2775        "L_aes_set_key_c_32_loop:\n\t"
2776            "addi   %[rk], %[rk], 32\n\t"
2777            /* Permute key. */
2778            AES_SUB_ROT_WORD_RCON(s2)
2779            "xor    t1, t1, t0\n\t"
2780            "xor    t2, t2, t1\n\t"
2781            "xor    t3, t3, t2\n\t"
2782            /* Store half round key. */
2783            "sw     t0,  0(%[rk])\n\t"
2784            "sw     t1,  4(%[rk])\n\t"
2785            "sw     t2,  8(%[rk])\n\t"
2786            "sw     t3, 12(%[rk])\n\t"
2787
2788            "addi   a5, a5, 4\n\t"
2789            "addi   a4, a4, -1\n\t"
2790            "beqz   a4, L_aes_set_key_c_32_done\n\t"
2791
2792            AES_SUB_WORD(t3)
2793            "xor    a7, a7, a6\n\t"
2794            "xor    s1, s1, a7\n\t"
2795            "xor    s2, s2, s1\n\t"
2796            /* Store second half round key. */
2797            "sw     a6, 16(%[rk])\n\t"
2798            "sw     a7, 20(%[rk])\n\t"
2799            "sw     s1, 24(%[rk])\n\t"
2800            "sw     s2, 28(%[rk])\n\t"
2801
2802            "beqz   x0, L_aes_set_key_c_32_loop\n\t"
2803        "L_aes_set_key_c_32_done:\n\t"
2804
2805            : [rk] "+r" (rk)
2806            : [key] "r" (key), [te] "r" (Te), [rcon] "r" (rcon)
2807            : "memory", "t0", "t1", "t2", "t3", "t4", "t5", "t6", "a4", "a5",
2808              "a6", "a7", "s1", "s2"
2809        );
2810        break;
2811#endif /* 256 */
2812    }
2813
2814#if defined(HAVE_AES_DECRYPT)
2815
2816#define INV_MIXCOL(rki)                     \
2817            "srli   t5, " #rki ", 22\n\t"   \
2818            "andi   t5, t5, 0x3fc\n\t"      \
2819            "add    t5, t5, %[te1]\n\t"     \
2820            "lbu    t5, (t5)\n\t"           \
2821            "slli   t5, t5, 2\n\t"          \
2822            "add    t5, t5, %[td]\n\t"      \
2823            "lw     t6, (t5)\n\t"           \
2824            "srli   t5, " #rki ", 14\n\t"   \
2825            "andi   t5, t5, 0x3fc\n\t"      \
2826            "add    t5, t5, %[te1]\n\t"     \
2827            "lbu    t5, (t5)\n\t"           \
2828            "slli   t5, t5, 2\n\t"          \
2829            "add    t5, t5, %[td]\n\t"      \
2830            "addi   t5, t5, 0x400\n\t"      \
2831            "lw     t5, (t5)\n\t"           \
2832            "xor    t6, t6, t5\n\t"         \
2833            "srli   t5, " #rki ", 6\n\t"    \
2834            "andi   t5, t5, 0x3fc\n\t"      \
2835            "add    t5, t5, %[te1]\n\t"     \
2836            "lbu    t5, (t5)\n\t"           \
2837            "slli   t5, t5, 2\n\t"          \
2838            "add    t5, t5, %[td]\n\t"      \
2839            "li     t4, 0x800\n\t"          \
2840            "add    t5, t5, t4\n\t"         \
2841            "lw     t5, (t5)\n\t"           \
2842            "xor    t6, t6, t5\n\t"         \
2843            "andi   t5, " #rki ", 0xff\n\t" \
2844            "slli   t5, t5, 2\n\t"          \
2845            "add    t5, t5, %[te1]\n\t"     \
2846            "lbu    t5, (t5)\n\t"           \
2847            "slli   t5, t5, 2\n\t"          \
2848            "add    t5, t5, %[td]\n\t"      \
2849            "li     t4, 0xc00\n\t"          \
2850            "add    t5, t5, t4\n\t"         \
2851            "lw     t5, (t5)\n\t"           \
2852            "xor    " #rki ", t6, t5\n\t"
2853
2854    if (dir == AES_DECRYPTION) {
2855        int r = aes->rounds;
2856        rk = aes->key;
2857
2858        __asm__ __volatile__ (
2859            /* Change key schedule for decryption. */
2860            "slli   s1, %[r], 4\n\t"
2861            "add    s1, s1, %[rk]\n\t"
2862            "srli   %[r], %[r], 1\n\t"
2863            "addi   %[r], %[r], -1\n\t"
2864
2865            /* Swap first two rounds. */
2866            "ld     t0, 0(%[rk])\n\t"
2867            "ld     t1, 8(%[rk])\n\t"
2868            "ld     t2, 0(s1)\n\t"
2869            "ld     t3, 8(s1)\n\t"
2870            "sd     t0, 0(s1)\n\t"
2871            "sd     t1, 8(s1)\n\t"
2872            "sd     t2, 0(%[rk])\n\t"
2873            "sd     t3, 8(%[rk])\n\t"
2874
2875       "L_aes_set_key_inv_mixcol_loop:\n\t"
2876            "addi   %[rk], %[rk], 16\n\t"
2877            "addi   s1, s1, -16\n\t"
2878
2879            "lw     t0,  0(%[rk])\n\t"
2880            "lw     t1,  4(%[rk])\n\t"
2881            "lw     t2,  8(%[rk])\n\t"
2882            "lw     t3, 12(%[rk])\n\t"
2883            "lw     a4,  0(s1)\n\t"
2884            "lw     a5,  4(s1)\n\t"
2885            "lw     a6,  8(s1)\n\t"
2886            "lw     a7, 12(s1)\n\t"
2887
2888            INV_MIXCOL(t0)
2889            INV_MIXCOL(t1)
2890            INV_MIXCOL(t2)
2891            INV_MIXCOL(t3)
2892            INV_MIXCOL(a4)
2893            INV_MIXCOL(a5)
2894            INV_MIXCOL(a6)
2895            INV_MIXCOL(a7)
2896
2897            "sw     t0,  0(s1)\n\t"
2898            "sw     t1,  4(s1)\n\t"
2899            "sw     t2,  8(s1)\n\t"
2900            "sw     t3, 12(s1)\n\t"
2901            "sw     a4,  0(%[rk])\n\t"
2902            "sw     a5,  4(%[rk])\n\t"
2903            "sw     a6,  8(%[rk])\n\t"
2904            "sw     a7, 12(%[rk])\n\t"
2905
2906            "addi   %[r], %[r], -1\n\t"
2907            "bnez   %[r], L_aes_set_key_inv_mixcol_loop\n\t"
2908
2909            "lw     t0, 16(%[rk])\n\t"
2910            "lw     t1, 20(%[rk])\n\t"
2911            "lw     t2, 24(%[rk])\n\t"
2912            "lw     t3, 28(%[rk])\n\t"
2913
2914            INV_MIXCOL(t0)
2915            INV_MIXCOL(t1)
2916            INV_MIXCOL(t2)
2917            INV_MIXCOL(t3)
2918
2919            "sw     t0, 16(%[rk])\n\t"
2920            "sw     t1, 20(%[rk])\n\t"
2921            "sw     t2, 24(%[rk])\n\t"
2922            "sw     t3, 28(%[rk])\n\t"
2923
2924            : [rk] "+r" (rk), [r] "+r" (r)
2925            : [td] "r" (Td), [te1] "r" (Te[1])
2926            : "memory", "t0", "t1", "t2", "t3", "t4", "t5", "t6", "a4", "a5",
2927              "a6", "a7", "s1"
2928       );
2929   }
2930#endif /* HAVE_AES_DECRYPT */
2931}
2932
2933/* Set the key and/or IV into the AES object.
2934 *
2935 * Creates the key schedule from the key.
2936 * Uses Cryptographic instructions.
2937 *
2938 * @param [in] aes     AES object.
2939 * @param [in] key     Secret key to use.
2940 * @param [in] keyLen  Length of key in bytes.
2941 * @param [in] iv      Initialization Vector (IV) to use. May be NULL.
2942 * @param [in] dir     Direction of crypt: AES_ENCRYPT, AES_DECRYPT.
2943 * @return  0 on success.
2944 * @return  BAD_FUNC_ARG when aes or key is NULL.
2945 * @return  BAD_FUNC_ARG when keyLen/dir is not supported or valid.
2946 */
2947int wc_AesSetKey(Aes* aes, const byte* key, word32 keyLen, const byte* iv,
2948    int dir)
2949{
2950    int ret = 0;
2951
2952    /* Validate parameters. */
2953    if (aes == NULL) {
2954        ret = BAD_FUNC_ARG;
2955    }
2956    /* Check key size is supported by AES object. */
2957    if ((ret == 0) && (keyLen > (word32)sizeof(aes->key))) {
2958        ret = BAD_FUNC_ARG;
2959    }
2960
2961    if (ret == 0) {
2962        /* Check key length is supported. */
2963        switch (keyLen) {
2964    #if defined(AES_MAX_KEY_SIZE) && (AES_MAX_KEY_SIZE >= 128) && \
2965        defined(WOLFSSL_AES_128)
2966        case 16:
2967    #endif
2968    #if defined(AES_MAX_KEY_SIZE) && (AES_MAX_KEY_SIZE >= 192) && \
2969        defined(WOLFSSL_AES_192)
2970        case 24:
2971    #endif
2972    #if defined(AES_MAX_KEY_SIZE) && (AES_MAX_KEY_SIZE >= 256) && \
2973        defined(WOLFSSL_AES_256)
2974        case 32:
2975    #endif
2976            break;
2977        default:
2978            ret = BAD_FUNC_ARG;
2979        }
2980    }
2981#ifndef HAVE_AES_DECRYPT
2982    if ((ret == 0) && (dir == AES_DECRYPTION)) {
2983        ret = BAD_FUNC_ARG;
2984    }
2985#endif
2986
2987    if (ret == 0) {
2988        /* Initialize fields. */
2989    #if defined(WOLFSSL_AES_COUNTER) || defined(WOLFSSL_AES_CFB) || \
2990        defined(WOLFSSL_AES_OFB) || defined(WOLFSSL_AES_XTS)
2991        aes->left = 0;
2992    #endif
2993        aes->keylen = (int)keyLen;
2994        aes->rounds = (keyLen / 4) + 6;
2995
2996        /* Compute the key schedule. */
2997        AesSetKey_C(aes, key, keyLen, dir);
2998
2999        /* Set the IV. */
3000        ret = wc_AesSetIV(aes, iv);
3001    }
3002
3003    return ret;
3004}
3005
3006#if defined(WOLFSSL_AES_DIRECT) || defined(HAVE_AES_CBC) || \
3007    defined(HAVE_AESGCM) || defined(HAVE_AESCCM)
3008/* Encrypt a block using AES.
3009 *
3010 * @param [in]  aes  AES object.
3011 * @param [in]  in   Block to encrypt.
3012 * @param [out] out  Encrypted block.
3013 */
3014static void wc_AesEncrypt(Aes* aes, const byte* in, byte* out)
3015{
3016/* A round of encryption from set 2 to 1 registers. */
3017#define ENC_ROUND_T_S_ASM(o)                    \
3018        "srliw      t0, a4, 24\n\t"             \
3019        "srliw      t1, a5, 24\n\t"             \
3020        "srliw      t2, a6, 24\n\t"             \
3021        "srliw      t3, a7, 24\n\t"             \
3022        "slliw      t0, t0, 2\n\t"              \
3023        "slliw      t1, t1, 2\n\t"              \
3024        "slliw      t2, t2, 2\n\t"              \
3025        "slliw      t3, t3, 2\n\t"              \
3026        "add        t0, t0, %[te]\n\t"          \
3027        "add        t1, t1, %[te]\n\t"          \
3028        "add        t2, t2, %[te]\n\t"          \
3029        "add        t3, t3, %[te]\n\t"          \
3030        "lw         t5, (t0)\n\t"               \
3031        "lw         t6, (t1)\n\t"               \
3032        "lw         s1, (t2)\n\t"               \
3033        "lw         s2, (t3)\n\t"               \
3034                                                \
3035        "addi       t4, %[te], 0x400\n\t"       \
3036        "srliw      t0, a5, 14\n\t"             \
3037        "srliw      t1, a6, 14\n\t"             \
3038        "srliw      t2, a7, 14\n\t"             \
3039        "srliw      t3, a4, 14\n\t"             \
3040        "andi       t0, t0, 0x3fc\n\t"          \
3041        "andi       t1, t1, 0x3fc\n\t"          \
3042        "andi       t2, t2, 0x3fc\n\t"          \
3043        "andi       t3, t3, 0x3fc\n\t"          \
3044        "add        t0, t0, t4\n\t"             \
3045        "add        t1, t1, t4\n\t"             \
3046        "add        t2, t2, t4\n\t"             \
3047        "add        t3, t3, t4\n\t"             \
3048        "lw         t0, (t0)\n\t"               \
3049        "lw         t1, (t1)\n\t"               \
3050        "lw         t2, (t2)\n\t"               \
3051        "lw         t3, (t3)\n\t"               \
3052        "xor        t5, t5, t0\n\t"             \
3053        "xor        t6, t6, t1\n\t"             \
3054        "xor        s1, s1, t2\n\t"             \
3055        "xor        s2, s2, t3\n\t"             \
3056                                                \
3057        "addi       t4, t4, 0x400\n\t"          \
3058        "srliw      t0, a6, 6\n\t"              \
3059        "srliw      t1, a7, 6\n\t"              \
3060        "srliw      t2, a4, 6\n\t"              \
3061        "srliw      t3, a5, 6\n\t"              \
3062        "andi       t0, t0, 0x3fc\n\t"          \
3063        "andi       t1, t1, 0x3fc\n\t"          \
3064        "andi       t2, t2, 0x3fc\n\t"          \
3065        "andi       t3, t3, 0x3fc\n\t"          \
3066        "add        t0, t0, t4\n\t"             \
3067        "add        t1, t1, t4\n\t"             \
3068        "add        t2, t2, t4\n\t"             \
3069        "add        t3, t3, t4\n\t"             \
3070        "lw         t0, (t0)\n\t"               \
3071        "lw         t1, (t1)\n\t"               \
3072        "lw         t2, (t2)\n\t"               \
3073        "lw         t3, (t3)\n\t"               \
3074        "xor        t5, t5, t0\n\t"             \
3075        "xor        t6, t6, t1\n\t"             \
3076        "xor        s1, s1, t2\n\t"             \
3077        "xor        s2, s2, t3\n\t"             \
3078                                                \
3079        "addi       t4, t4, 0x400\n\t"          \
3080        "andi       t0, a7, 0xff\n\t"           \
3081        "andi       t1, a4, 0xff\n\t"           \
3082        "andi       t2, a5, 0xff\n\t"           \
3083        "andi       t3, a6, 0xff\n\t"           \
3084        "slliw      t0, t0, 2\n\t"              \
3085        "slliw      t1, t1, 2\n\t"              \
3086        "slliw      t2, t2, 2\n\t"              \
3087        "slliw      t3, t3, 2\n\t"              \
3088        "add        t0, t0, t4\n\t"             \
3089        "add        t1, t1, t4\n\t"             \
3090        "add        t2, t2, t4\n\t"             \
3091        "add        t3, t3, t4\n\t"             \
3092        "lw         t0, (t0)\n\t"               \
3093        "lw         t1, (t1)\n\t"               \
3094        "lw         t2, (t2)\n\t"               \
3095        "lw         t3, (t3)\n\t"               \
3096        "xor        t5, t5, t0\n\t"             \
3097        "xor        t6, t6, t1\n\t"             \
3098        "xor        s1, s1, t2\n\t"             \
3099        "xor        s2, s2, t3\n\t"             \
3100                                                \
3101        "lw         t0, " #o "(%[rk])\n\t"      \
3102        "lw         t1, " #o "+4(%[rk])\n\t"    \
3103        "lw         t2, " #o "+8(%[rk])\n\t"    \
3104        "lw         t3, " #o "+12(%[rk])\n\t"   \
3105        "xor        t5, t5, t0\n\t"             \
3106        "xor        t6, t6, t1\n\t"             \
3107        "xor        s1, s1, t2\n\t"             \
3108        "xor        s2, s2, t3\n\t"
3109
3110/* A round of encryption from set 1 to 2 registers. */
3111#define ENC_ROUND_S_T_ASM(o)                    \
3112        "srliw      t0, t5, 24\n\t"             \
3113        "srliw      t1, t6, 24\n\t"             \
3114        "srliw      t2, s1, 24\n\t"             \
3115        "srliw      t3, s2, 24\n\t"             \
3116        "slliw      t0, t0, 2\n\t"              \
3117        "slliw      t1, t1, 2\n\t"              \
3118        "slliw      t2, t2, 2\n\t"              \
3119        "slliw      t3, t3, 2\n\t"              \
3120        "add        t0, t0, %[te]\n\t"          \
3121        "add        t1, t1, %[te]\n\t"          \
3122        "add        t2, t2, %[te]\n\t"          \
3123        "add        t3, t3, %[te]\n\t"          \
3124        "lw         a4, (t0)\n\t"               \
3125        "lw         a5, (t1)\n\t"               \
3126        "lw         a6, (t2)\n\t"               \
3127        "lw         a7, (t3)\n\t"               \
3128                                                \
3129        "addi       t4, %[te], 0x400\n\t"       \
3130        "srliw      t0, t6, 14\n\t"             \
3131        "srliw      t1, s1, 14\n\t"             \
3132        "srliw      t2, s2, 14\n\t"             \
3133        "srliw      t3, t5, 14\n\t"             \
3134        "andi       t0, t0, 0x3fc\n\t"          \
3135        "andi       t1, t1, 0x3fc\n\t"          \
3136        "andi       t2, t2, 0x3fc\n\t"          \
3137        "andi       t3, t3, 0x3fc\n\t"          \
3138        "add        t0, t0, t4\n\t"             \
3139        "add        t1, t1, t4\n\t"             \
3140        "add        t2, t2, t4\n\t"             \
3141        "add        t3, t3, t4\n\t"             \
3142        "lw         t0, (t0)\n\t"               \
3143        "lw         t1, (t1)\n\t"               \
3144        "lw         t2, (t2)\n\t"               \
3145        "lw         t3, (t3)\n\t"               \
3146        "xor        a4, a4, t0\n\t"             \
3147        "xor        a5, a5, t1\n\t"             \
3148        "xor        a6, a6, t2\n\t"             \
3149        "xor        a7, a7, t3\n\t"             \
3150                                                \
3151        "addi       t4, t4, 0x400\n\t"          \
3152        "srliw      t0, s1, 6\n\t"              \
3153        "srliw      t1, s2, 6\n\t"              \
3154        "srliw      t2, t5, 6\n\t"              \
3155        "srliw      t3, t6, 6\n\t"              \
3156        "andi       t0, t0, 0x3fc\n\t"          \
3157        "andi       t1, t1, 0x3fc\n\t"          \
3158        "andi       t2, t2, 0x3fc\n\t"          \
3159        "andi       t3, t3, 0x3fc\n\t"          \
3160        "add        t0, t0, t4\n\t"             \
3161        "add        t1, t1, t4\n\t"             \
3162        "add        t2, t2, t4\n\t"             \
3163        "add        t3, t3, t4\n\t"             \
3164        "lw         t0, (t0)\n\t"               \
3165        "lw         t1, (t1)\n\t"               \
3166        "lw         t2, (t2)\n\t"               \
3167        "lw         t3, (t3)\n\t"               \
3168        "xor        a4, a4, t0\n\t"             \
3169        "xor        a5, a5, t1\n\t"             \
3170        "xor        a6, a6, t2\n\t"             \
3171        "xor        a7, a7, t3\n\t"             \
3172                                                \
3173        "addi       t4, t4, 0x400\n\t"          \
3174        "andi       t0, s2, 0xff\n\t"           \
3175        "andi       t1, t5, 0xff\n\t"           \
3176        "andi       t2, t6, 0xff\n\t"           \
3177        "andi       t3, s1, 0xff\n\t"           \
3178        "slliw      t0, t0, 2\n\t"              \
3179        "slliw      t1, t1, 2\n\t"              \
3180        "slliw      t2, t2, 2\n\t"              \
3181        "slliw      t3, t3, 2\n\t"              \
3182        "add        t0, t0, t4\n\t"             \
3183        "add        t1, t1, t4\n\t"             \
3184        "add        t2, t2, t4\n\t"             \
3185        "add        t3, t3, t4\n\t"             \
3186        "lw         t0, (t0)\n\t"               \
3187        "lw         t1, (t1)\n\t"               \
3188        "lw         t2, (t2)\n\t"               \
3189        "lw         t3, (t3)\n\t"               \
3190        "xor        a4, a4, t0\n\t"             \
3191        "xor        a5, a5, t1\n\t"             \
3192        "xor        a6, a6, t2\n\t"             \
3193        "xor        a7, a7, t3\n\t"             \
3194                                                \
3195        "lw         t0, " #o "(%[rk])\n\t"      \
3196        "lw         t1, " #o "+4(%[rk])\n\t"    \
3197        "lw         t2, " #o "+8(%[rk])\n\t"    \
3198        "lw         t3, " #o "+12(%[rk])\n\t"   \
3199        "xor        a4, a4, t0\n\t"             \
3200        "xor        a5, a5, t1\n\t"             \
3201        "xor        a6, a6, t2\n\t"             \
3202        "xor        a7, a7, t3\n\t"
3203
3204    __asm__ __volatile__ (
3205#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
3206        /* Load 4 32-bit words in reverse byte order. */
3207        LOAD_WORD_REV(t0, 0, %[in])
3208        LOAD_WORD_REV(t1, 4, %[in])
3209        LOAD_WORD_REV(t2, 8, %[in])
3210        LOAD_WORD_REV(t3, 12, %[in])
3211#else
3212        "ld         t1,  0(%[in])\n\t"
3213        "ld         t3,  8(%[in])\n\t"
3214        REV8(REG_T1, REG_T1)
3215        REV8(REG_T3, REG_T3)
3216        "srli       t0, t1, 32\n\t"
3217        "srli       t2, t3, 32\n\t"
3218#endif
3219        "lw         a4,  0(%[rk])\n\t"
3220        "lw         a5,  4(%[rk])\n\t"
3221        "lw         a6,  8(%[rk])\n\t"
3222        "lw         a7, 12(%[rk])\n\t"
3223        /* AddRoundKey */
3224        "xor        a4, t0, a4\n\t"
3225        "xor        a5, t1, a5\n\t"
3226        "xor        a6, t2, a6\n\t"
3227        "xor        a7, t3, a7\n\t"
3228
3229        ENC_ROUND_T_S_ASM(16)
3230        ENC_ROUND_S_T_ASM(32)
3231        ENC_ROUND_T_S_ASM(48)
3232        ENC_ROUND_S_T_ASM(64)
3233        ENC_ROUND_T_S_ASM(80)
3234        ENC_ROUND_S_T_ASM(96)
3235        ENC_ROUND_T_S_ASM(112)
3236        ENC_ROUND_S_T_ASM(128)
3237        ENC_ROUND_T_S_ASM(144)
3238
3239        "li          t4, 5\n\t"
3240        "ble         %[r], t4, L_aes_encrypt_done\n\t"
3241        ENC_ROUND_S_T_ASM(160)
3242        ENC_ROUND_T_S_ASM(176)
3243
3244        "li          t4, 6\n\t"
3245        "ble         %[r], t4, L_aes_encrypt_done\n\t"
3246        ENC_ROUND_S_T_ASM(192)
3247        ENC_ROUND_T_S_ASM(208)
3248    "L_aes_encrypt_done:\n\t"
3249
3250        /* Last round. */
3251        "srliw      t0, s1, 6\n\t"
3252        "srliw      t1, s2, 6\n\t"
3253        "srliw      t2, t5, 6\n\t"
3254        "srliw      t3, t6, 6\n\t"
3255        "andi       t0, t0, 0x3fc\n\t"
3256        "andi       t1, t1, 0x3fc\n\t"
3257        "andi       t2, t2, 0x3fc\n\t"
3258        "andi       t3, t3, 0x3fc\n\t"
3259        "add        t0, t0, %[te]\n\t"
3260        "add        t1, t1, %[te]\n\t"
3261        "add        t2, t2, %[te]\n\t"
3262        "add        t3, t3, %[te]\n\t"
3263        "lw         a4, (t0)\n\t"
3264        "lw         a5, (t1)\n\t"
3265        "lw         a6, (t2)\n\t"
3266        "lw         a7, (t3)\n\t"
3267        "li         t4, 0x0000ff00\n\t"
3268        "and        a4, a4, t4\n\t"
3269        "and        a5, a5, t4\n\t"
3270        "and        a6, a6, t4\n\t"
3271        "and        a7, a7, t4\n\t"
3272
3273        "addi       t4, %[te], 0x400\n\t"
3274        "andi       t0, s2, 0xff\n\t"
3275        "andi       t1, t5, 0xff\n\t"
3276        "andi       t2, t6, 0xff\n\t"
3277        "andi       t3, s1, 0xff\n\t"
3278        "slli       t0, t0, 2\n\t"
3279        "slli       t1, t1, 2\n\t"
3280        "slli       t2, t2, 2\n\t"
3281        "slli       t3, t3, 2\n\t"
3282        "add        t0, t0, t4\n\t"
3283        "add        t1, t1, t4\n\t"
3284        "add        t2, t2, t4\n\t"
3285        "add        t3, t3, t4\n\t"
3286        "lbu        t0, (t0)\n\t"
3287        "lbu        t1, (t1)\n\t"
3288        "lbu        t2, (t2)\n\t"
3289        "lbu        t3, (t3)\n\t"
3290        "or         a4, a4, t0\n\t"
3291        "or         a5, a5, t1\n\t"
3292        "or         a6, a6, t2\n\t"
3293        "or         a7, a7, t3\n\t"
3294
3295        "addi       t4, t4, 0x400\n\t"
3296        "srliw      t0, t5, 24\n\t"
3297        "srliw      t1, t6, 24\n\t"
3298        "srliw      t2, s1, 24\n\t"
3299        "srliw      t3, s2, 24\n\t"
3300        "slli       t0, t0, 2\n\t"
3301        "slli       t1, t1, 2\n\t"
3302        "slli       t2, t2, 2\n\t"
3303        "slli       t3, t3, 2\n\t"
3304        "add        t0, t0, t4\n\t"
3305        "add        t1, t1, t4\n\t"
3306        "add        t2, t2, t4\n\t"
3307        "add        t3, t3, t4\n\t"
3308        "lw         t0, (t0)\n\t"
3309        "lw         t1, (t1)\n\t"
3310        "lw         t2, (t2)\n\t"
3311        "lw         t3, (t3)\n\t"
3312        "li         t4, 0xff000000\n\t"
3313        "and        t0, t0, t4\n\t"
3314        "and        t1, t1, t4\n\t"
3315        "and        t2, t2, t4\n\t"
3316        "and        t3, t3, t4\n\t"
3317        "or         a4, a4, t0\n\t"
3318        "or         a5, a5, t1\n\t"
3319        "or         a6, a6, t2\n\t"
3320        "or         a7, a7, t3\n\t"
3321
3322        "li         t4, 0xc00\n\t"
3323        "add        t4, %[te], t4\n\t"
3324        "srliw      t0, t6, 14\n\t"
3325        "srliw      t1, s1, 14\n\t"
3326        "srliw      t2, s2, 14\n\t"
3327        "srliw      t3, t5, 14\n\t"
3328        "andi       t0, t0, 0x3fc\n\t"
3329        "andi       t1, t1, 0x3fc\n\t"
3330        "andi       t2, t2, 0x3fc\n\t"
3331        "andi       t3, t3, 0x3fc\n\t"
3332        "add        t0, t0, t4\n\t"
3333        "add        t1, t1, t4\n\t"
3334        "add        t2, t2, t4\n\t"
3335        "add        t3, t3, t4\n\t"
3336        "lw         t0, (t0)\n\t"
3337        "lw         t1, (t1)\n\t"
3338        "lw         t2, (t2)\n\t"
3339        "lw         t3, (t3)\n\t"
3340        "li         t4, 0x00ff0000\n\t"
3341        "and        t0, t0, t4\n\t"
3342        "and        t1, t1, t4\n\t"
3343        "and        t2, t2, t4\n\t"
3344        "and        t3, t3, t4\n\t"
3345        "or         a4, a4, t0\n\t"
3346        "or         a5, a5, t1\n\t"
3347        "or         a6, a6, t2\n\t"
3348        "or         a7, a7, t3\n\t"
3349
3350        "slli       t4, %[r], 5\n\t"
3351        "add        t4, %[rk], t4\n\t"
3352        "lw         t0,  0(t4)\n\t"
3353        "lw         t1,  4(t4)\n\t"
3354        "lw         t2,  8(t4)\n\t"
3355        "lw         t3, 12(t4)\n\t"
3356        "xor        a4, a4, t0\n\t"
3357        "xor        a5, a5, t1\n\t"
3358        "xor        a6, a6, t2\n\t"
3359        "xor        a7, a7, t3\n\t"
3360
3361        /* Reverse byte in 32-bit words. */
3362#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
3363        STORE_WORD_REV(a4, 0, %[out])
3364        STORE_WORD_REV(a5, 4, %[out])
3365        STORE_WORD_REV(a6, 8, %[out])
3366        STORE_WORD_REV(a7, 12, %[out])
3367#elif !defined(WOLFSSL_RISCV_BIT_MANIPULATION)
3368        "slli        t0, a4, 32\n\t"
3369        "slli        t1, a5, 32\n\t"
3370        "slli        t2, a6, 32\n\t"
3371        "slli        t3, a7, 32\n\t"
3372        "srli        t1, t1, 32\n\t"
3373        "srli        t3, t3, 32\n\t"
3374        "or          t1, t1, t0\n\t"
3375        "or          t3, t3, t2\n\t"
3376        REV8(REG_T1, REG_T1)
3377        REV8(REG_T3, REG_T3)
3378        /* Write encrypted block to output. */
3379        "sd         t1,  0(%[out])\n\t"
3380        "sd         t3,  8(%[out])\n\t"
3381#else
3382        PACK(REG_T1, REG_A5, REG_A4)
3383        PACK(REG_T3, REG_A7, REG_A6)
3384        REV8(REG_T1, REG_T1)
3385        REV8(REG_T3, REG_T3)
3386        /* Write encrypted block to output. */
3387        "sd         t1,  0(%[out])\n\t"
3388        "sd         t3,  8(%[out])\n\t"
3389#endif
3390
3391        :
3392        : [in] "r" (in), [rk] "r" (aes->key), [te] "r" (Te),
3393          [r] "r" (aes->rounds >> 1), [out] "r" (out)
3394        : "memory", "t0", "t1", "t2", "t3", "t4",
3395          "a4", "a5", "a6", "a7",
3396          "t5", "t6", "s1", "s2"
3397    );
3398}
3399#endif /* WOLFSSL_AES_DIRECT || HAVE_AES_CBC || HAVE_AESGCM || HAVE_AESCCM */
3400
3401#if defined(WOLFSSL_AES_DIRECT) || defined(HAVE_AES_CBC)
3402#ifdef HAVE_AES_DECRYPT
3403/* AES byte decryption table. */
3404static const FLASH_QUALIFIER byte Td4[256] =
3405{
3406    0x52U, 0x09U, 0x6aU, 0xd5U, 0x30U, 0x36U, 0xa5U, 0x38U,
3407    0xbfU, 0x40U, 0xa3U, 0x9eU, 0x81U, 0xf3U, 0xd7U, 0xfbU,
3408    0x7cU, 0xe3U, 0x39U, 0x82U, 0x9bU, 0x2fU, 0xffU, 0x87U,
3409    0x34U, 0x8eU, 0x43U, 0x44U, 0xc4U, 0xdeU, 0xe9U, 0xcbU,
3410    0x54U, 0x7bU, 0x94U, 0x32U, 0xa6U, 0xc2U, 0x23U, 0x3dU,
3411    0xeeU, 0x4cU, 0x95U, 0x0bU, 0x42U, 0xfaU, 0xc3U, 0x4eU,
3412    0x08U, 0x2eU, 0xa1U, 0x66U, 0x28U, 0xd9U, 0x24U, 0xb2U,
3413    0x76U, 0x5bU, 0xa2U, 0x49U, 0x6dU, 0x8bU, 0xd1U, 0x25U,
3414    0x72U, 0xf8U, 0xf6U, 0x64U, 0x86U, 0x68U, 0x98U, 0x16U,
3415    0xd4U, 0xa4U, 0x5cU, 0xccU, 0x5dU, 0x65U, 0xb6U, 0x92U,
3416    0x6cU, 0x70U, 0x48U, 0x50U, 0xfdU, 0xedU, 0xb9U, 0xdaU,
3417    0x5eU, 0x15U, 0x46U, 0x57U, 0xa7U, 0x8dU, 0x9dU, 0x84U,
3418    0x90U, 0xd8U, 0xabU, 0x00U, 0x8cU, 0xbcU, 0xd3U, 0x0aU,
3419    0xf7U, 0xe4U, 0x58U, 0x05U, 0xb8U, 0xb3U, 0x45U, 0x06U,
3420    0xd0U, 0x2cU, 0x1eU, 0x8fU, 0xcaU, 0x3fU, 0x0fU, 0x02U,
3421    0xc1U, 0xafU, 0xbdU, 0x03U, 0x01U, 0x13U, 0x8aU, 0x6bU,
3422    0x3aU, 0x91U, 0x11U, 0x41U, 0x4fU, 0x67U, 0xdcU, 0xeaU,
3423    0x97U, 0xf2U, 0xcfU, 0xceU, 0xf0U, 0xb4U, 0xe6U, 0x73U,
3424    0x96U, 0xacU, 0x74U, 0x22U, 0xe7U, 0xadU, 0x35U, 0x85U,
3425    0xe2U, 0xf9U, 0x37U, 0xe8U, 0x1cU, 0x75U, 0xdfU, 0x6eU,
3426    0x47U, 0xf1U, 0x1aU, 0x71U, 0x1dU, 0x29U, 0xc5U, 0x89U,
3427    0x6fU, 0xb7U, 0x62U, 0x0eU, 0xaaU, 0x18U, 0xbeU, 0x1bU,
3428    0xfcU, 0x56U, 0x3eU, 0x4bU, 0xc6U, 0xd2U, 0x79U, 0x20U,
3429    0x9aU, 0xdbU, 0xc0U, 0xfeU, 0x78U, 0xcdU, 0x5aU, 0xf4U,
3430    0x1fU, 0xddU, 0xa8U, 0x33U, 0x88U, 0x07U, 0xc7U, 0x31U,
3431    0xb1U, 0x12U, 0x10U, 0x59U, 0x27U, 0x80U, 0xecU, 0x5fU,
3432    0x60U, 0x51U, 0x7fU, 0xa9U, 0x19U, 0xb5U, 0x4aU, 0x0dU,
3433    0x2dU, 0xe5U, 0x7aU, 0x9fU, 0x93U, 0xc9U, 0x9cU, 0xefU,
3434    0xa0U, 0xe0U, 0x3bU, 0x4dU, 0xaeU, 0x2aU, 0xf5U, 0xb0U,
3435    0xc8U, 0xebU, 0xbbU, 0x3cU, 0x83U, 0x53U, 0x99U, 0x61U,
3436    0x17U, 0x2bU, 0x04U, 0x7eU, 0xbaU, 0x77U, 0xd6U, 0x26U,
3437    0xe1U, 0x69U, 0x14U, 0x63U, 0x55U, 0x21U, 0x0cU, 0x7dU,
3438};
3439
3440/* Decrypt a block using AES.
3441 *
3442 * @param [in]  aes  AES object.
3443 * @param [in]  in   Block to decrypt.
3444 * @param [out] out  Decrypted block.
3445 */
3446static void wc_AesDecrypt(Aes* aes, const byte* in, byte* out)
3447{
3448/* A round of decryption from set 2 to 1 registers. */
3449#define DEC_ROUND_T_S_ASM(o)                    \
3450        "srliw      t0, a4, 24\n\t"             \
3451        "srliw      t1, a5, 24\n\t"             \
3452        "srliw      t2, a6, 24\n\t"             \
3453        "srliw      t3, a7, 24\n\t"             \
3454        "slliw      t0, t0, 2\n\t"              \
3455        "slliw      t1, t1, 2\n\t"              \
3456        "slliw      t2, t2, 2\n\t"              \
3457        "slliw      t3, t3, 2\n\t"              \
3458        "add        t0, t0, %[td]\n\t"          \
3459        "add        t1, t1, %[td]\n\t"          \
3460        "add        t2, t2, %[td]\n\t"          \
3461        "add        t3, t3, %[td]\n\t"          \
3462        "lw         t5, (t0)\n\t"               \
3463        "lw         t6, (t1)\n\t"               \
3464        "lw         s1, (t2)\n\t"               \
3465        "lw         s2, (t3)\n\t"               \
3466                                                \
3467        "addi       t4, %[td], 0x400\n\t"       \
3468        "srliw      t0, a7, 14\n\t"             \
3469        "srliw      t1, a4, 14\n\t"             \
3470        "srliw      t2, a5, 14\n\t"             \
3471        "srliw      t3, a6, 14\n\t"             \
3472        "andi       t0, t0, 0x3fc\n\t"          \
3473        "andi       t1, t1, 0x3fc\n\t"          \
3474        "andi       t2, t2, 0x3fc\n\t"          \
3475        "andi       t3, t3, 0x3fc\n\t"          \
3476        "add        t0, t0, t4\n\t"             \
3477        "add        t1, t1, t4\n\t"             \
3478        "add        t2, t2, t4\n\t"             \
3479        "add        t3, t3, t4\n\t"             \
3480        "lw         t0, (t0)\n\t"               \
3481        "lw         t1, (t1)\n\t"               \
3482        "lw         t2, (t2)\n\t"               \
3483        "lw         t3, (t3)\n\t"               \
3484        "xor        t5, t5, t0\n\t"             \
3485        "xor        t6, t6, t1\n\t"             \
3486        "xor        s1, s1, t2\n\t"             \
3487        "xor        s2, s2, t3\n\t"             \
3488                                                \
3489        "addi       t4, t4, 0x400\n\t"          \
3490        "srliw      t0, a6, 6\n\t"              \
3491        "srliw      t1, a7, 6\n\t"              \
3492        "srliw      t2, a4, 6\n\t"              \
3493        "srliw      t3, a5, 6\n\t"              \
3494        "andi       t0, t0, 0x3fc\n\t"          \
3495        "andi       t1, t1, 0x3fc\n\t"          \
3496        "andi       t2, t2, 0x3fc\n\t"          \
3497        "andi       t3, t3, 0x3fc\n\t"          \
3498        "add        t0, t0, t4\n\t"             \
3499        "add        t1, t1, t4\n\t"             \
3500        "add        t2, t2, t4\n\t"             \
3501        "add        t3, t3, t4\n\t"             \
3502        "lw         t0, (t0)\n\t"               \
3503        "lw         t1, (t1)\n\t"               \
3504        "lw         t2, (t2)\n\t"               \
3505        "lw         t3, (t3)\n\t"               \
3506        "xor        t5, t5, t0\n\t"             \
3507        "xor        t6, t6, t1\n\t"             \
3508        "xor        s1, s1, t2\n\t"             \
3509        "xor        s2, s2, t3\n\t"             \
3510                                                \
3511        "addi       t4, t4, 0x400\n\t"          \
3512        "andi       t0, a5, 0xff\n\t"           \
3513        "andi       t1, a6, 0xff\n\t"           \
3514        "andi       t2, a7, 0xff\n\t"           \
3515        "andi       t3, a4, 0xff\n\t"           \
3516        "slliw      t0, t0, 2\n\t"              \
3517        "slliw      t1, t1, 2\n\t"              \
3518        "slliw      t2, t2, 2\n\t"              \
3519        "slliw      t3, t3, 2\n\t"              \
3520        "add        t0, t0, t4\n\t"             \
3521        "add        t1, t1, t4\n\t"             \
3522        "add        t2, t2, t4\n\t"             \
3523        "add        t3, t3, t4\n\t"             \
3524        "lw         t0, (t0)\n\t"               \
3525        "lw         t1, (t1)\n\t"               \
3526        "lw         t2, (t2)\n\t"               \
3527        "lw         t3, (t3)\n\t"               \
3528        "xor        t5, t5, t0\n\t"             \
3529        "xor        t6, t6, t1\n\t"             \
3530        "xor        s1, s1, t2\n\t"             \
3531        "xor        s2, s2, t3\n\t"             \
3532                                                \
3533        "lw         t0, " #o "(%[rk])\n\t"      \
3534        "lw         t1, " #o "+4(%[rk])\n\t"    \
3535        "lw         t2, " #o "+8(%[rk])\n\t"    \
3536        "lw         t3, " #o "+12(%[rk])\n\t"   \
3537        "xor        t5, t5, t0\n\t"             \
3538        "xor        t6, t6, t1\n\t"             \
3539        "xor        s1, s1, t2\n\t"             \
3540        "xor        s2, s2, t3\n\t"
3541
3542/* A round of decryption from set 1 to 2 registers. */
3543#define DEC_ROUND_S_T_ASM(o)                    \
3544        "srliw      t0, t5, 24\n\t"             \
3545        "srliw      t1, t6, 24\n\t"             \
3546        "srliw      t2, s1, 24\n\t"             \
3547        "srliw      t3, s2, 24\n\t"             \
3548        "slliw      t0, t0, 2\n\t"              \
3549        "slliw      t1, t1, 2\n\t"              \
3550        "slliw      t2, t2, 2\n\t"              \
3551        "slliw      t3, t3, 2\n\t"              \
3552        "add        t0, t0, %[td]\n\t"          \
3553        "add        t1, t1, %[td]\n\t"          \
3554        "add        t2, t2, %[td]\n\t"          \
3555        "add        t3, t3, %[td]\n\t"          \
3556        "lw         a4, (t0)\n\t"               \
3557        "lw         a5, (t1)\n\t"               \
3558        "lw         a6, (t2)\n\t"               \
3559        "lw         a7, (t3)\n\t"               \
3560                                                \
3561        "addi       t4, %[td], 0x400\n\t"       \
3562        "srliw      t0, s2, 14\n\t"             \
3563        "srliw      t1, t5, 14\n\t"             \
3564        "srliw      t2, t6, 14\n\t"             \
3565        "srliw      t3, s1, 14\n\t"             \
3566        "andi       t0, t0, 0x3fc\n\t"          \
3567        "andi       t1, t1, 0x3fc\n\t"          \
3568        "andi       t2, t2, 0x3fc\n\t"          \
3569        "andi       t3, t3, 0x3fc\n\t"          \
3570        "add        t0, t0, t4\n\t"             \
3571        "add        t1, t1, t4\n\t"             \
3572        "add        t2, t2, t4\n\t"             \
3573        "add        t3, t3, t4\n\t"             \
3574        "lw         t0, (t0)\n\t"               \
3575        "lw         t1, (t1)\n\t"               \
3576        "lw         t2, (t2)\n\t"               \
3577        "lw         t3, (t3)\n\t"               \
3578        "xor        a4, a4, t0\n\t"             \
3579        "xor        a5, a5, t1\n\t"             \
3580        "xor        a6, a6, t2\n\t"             \
3581        "xor        a7, a7, t3\n\t"             \
3582                                                \
3583        "addi       t4, t4, 0x400\n\t"          \
3584        "srliw      t0, s1, 6\n\t"              \
3585        "srliw      t1, s2, 6\n\t"              \
3586        "srliw      t2, t5, 6\n\t"              \
3587        "srliw      t3, t6, 6\n\t"              \
3588        "andi       t0, t0, 0x3fc\n\t"          \
3589        "andi       t1, t1, 0x3fc\n\t"          \
3590        "andi       t2, t2, 0x3fc\n\t"          \
3591        "andi       t3, t3, 0x3fc\n\t"          \
3592        "add        t0, t0, t4\n\t"             \
3593        "add        t1, t1, t4\n\t"             \
3594        "add        t2, t2, t4\n\t"             \
3595        "add        t3, t3, t4\n\t"             \
3596        "lw         t0, (t0)\n\t"               \
3597        "lw         t1, (t1)\n\t"               \
3598        "lw         t2, (t2)\n\t"               \
3599        "lw         t3, (t3)\n\t"               \
3600        "xor        a4, a4, t0\n\t"             \
3601        "xor        a5, a5, t1\n\t"             \
3602        "xor        a6, a6, t2\n\t"             \
3603        "xor        a7, a7, t3\n\t"             \
3604                                                \
3605        "addi       t4, t4, 0x400\n\t"          \
3606        "andi       t0, t6, 0xff\n\t"           \
3607        "andi       t1, s1, 0xff\n\t"           \
3608        "andi       t2, s2, 0xff\n\t"           \
3609        "andi       t3, t5, 0xff\n\t"           \
3610        "slliw      t0, t0, 2\n\t"              \
3611        "slliw      t1, t1, 2\n\t"              \
3612        "slliw      t2, t2, 2\n\t"              \
3613        "slliw      t3, t3, 2\n\t"              \
3614        "add        t0, t0, t4\n\t"             \
3615        "add        t1, t1, t4\n\t"             \
3616        "add        t2, t2, t4\n\t"             \
3617        "add        t3, t3, t4\n\t"             \
3618        "lw         t0, (t0)\n\t"               \
3619        "lw         t1, (t1)\n\t"               \
3620        "lw         t2, (t2)\n\t"               \
3621        "lw         t3, (t3)\n\t"               \
3622        "xor        a4, a4, t0\n\t"             \
3623        "xor        a5, a5, t1\n\t"             \
3624        "xor        a6, a6, t2\n\t"             \
3625        "xor        a7, a7, t3\n\t"             \
3626                                                \
3627        "lw         t0, " #o "(%[rk])\n\t"      \
3628        "lw         t1, " #o "+4(%[rk])\n\t"    \
3629        "lw         t2, " #o "+8(%[rk])\n\t"    \
3630        "lw         t3, " #o "+12(%[rk])\n\t"   \
3631        "xor        a4, a4, t0\n\t"             \
3632        "xor        a5, a5, t1\n\t"             \
3633        "xor        a6, a6, t2\n\t"             \
3634        "xor        a7, a7, t3\n\t"
3635
3636    __asm__ __volatile__ (
3637#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
3638        /* Load 4 32-bit words in reverse byte order. */
3639        LOAD_WORD_REV(t0, 0, %[in])
3640        LOAD_WORD_REV(t1, 4, %[in])
3641        LOAD_WORD_REV(t2, 8, %[in])
3642        LOAD_WORD_REV(t3, 12, %[in])
3643#else
3644        "ld         t1,  0(%[in])\n\t"
3645        "ld         t3,  8(%[in])\n\t"
3646        REV8(REG_T1, REG_T1)
3647        REV8(REG_T3, REG_T3)
3648        "srli       t0, t1, 32\n\t"
3649        "srli       t2, t3, 32\n\t"
3650#endif
3651        "lw         a4,  0(%[rk])\n\t"
3652        "lw         a5,  4(%[rk])\n\t"
3653        "lw         a6,  8(%[rk])\n\t"
3654        "lw         a7, 12(%[rk])\n\t"
3655        /* AddRoundKey */
3656        "xor        a4, t0, a4\n\t"
3657        "xor        a5, t1, a5\n\t"
3658        "xor        a6, t2, a6\n\t"
3659        "xor        a7, t3, a7\n\t"
3660
3661        DEC_ROUND_T_S_ASM(16)
3662        DEC_ROUND_S_T_ASM(32)
3663        DEC_ROUND_T_S_ASM(48)
3664        DEC_ROUND_S_T_ASM(64)
3665        DEC_ROUND_T_S_ASM(80)
3666        DEC_ROUND_S_T_ASM(96)
3667        DEC_ROUND_T_S_ASM(112)
3668        DEC_ROUND_S_T_ASM(128)
3669        DEC_ROUND_T_S_ASM(144)
3670
3671        "li          t4, 5\n\t"
3672        "ble         %[r], t4, L_aes_decrypt_done\n\t"
3673        DEC_ROUND_S_T_ASM(160)
3674        DEC_ROUND_T_S_ASM(176)
3675
3676        "li          t4, 6\n\t"
3677        "ble         %[r], t4, L_aes_decrypt_done\n\t"
3678        DEC_ROUND_S_T_ASM(192)
3679        DEC_ROUND_T_S_ASM(208)
3680    "L_aes_decrypt_done:\n\t"
3681
3682        /* Last round. */
3683        "srliw      t0, t5, 24\n\t"
3684        "srliw      t1, t6, 24\n\t"
3685        "srliw      t2, s1, 24\n\t"
3686        "srliw      t3, s2, 24\n\t"
3687        "add        t0, t0, %[td4]\n\t"
3688        "add        t1, t1, %[td4]\n\t"
3689        "add        t2, t2, %[td4]\n\t"
3690        "add        t3, t3, %[td4]\n\t"
3691        "lbu        a4, (t0)\n\t"
3692        "lbu        a5, (t1)\n\t"
3693        "lbu        a6, (t2)\n\t"
3694        "lbu        a7, (t3)\n\t"
3695        "slli       a4, a4, 24\n\t"
3696        "slli       a5, a5, 24\n\t"
3697        "slli       a6, a6, 24\n\t"
3698        "slli       a7, a7, 24\n\t"
3699
3700        "srliw      t0, s2, 16\n\t"
3701        "srliw      t1, t5, 16\n\t"
3702        "srliw      t2, t6, 16\n\t"
3703        "srliw      t3, s1, 16\n\t"
3704        "andi       t0, t0, 0xff\n\t"
3705        "andi       t1, t1, 0xff\n\t"
3706        "andi       t2, t2, 0xff\n\t"
3707        "andi       t3, t3, 0xff\n\t"
3708        "add        t0, t0, %[td4]\n\t"
3709        "add        t1, t1, %[td4]\n\t"
3710        "add        t2, t2, %[td4]\n\t"
3711        "add        t3, t3, %[td4]\n\t"
3712        "lbu        t0, (t0)\n\t"
3713        "lbu        t1, (t1)\n\t"
3714        "lbu        t2, (t2)\n\t"
3715        "lbu        t3, (t3)\n\t"
3716        "slli       t0, t0, 16\n\t"
3717        "slli       t1, t1, 16\n\t"
3718        "slli       t2, t2, 16\n\t"
3719        "slli       t3, t3, 16\n\t"
3720        "or         a4, a4, t0\n\t"
3721        "or         a5, a5, t1\n\t"
3722        "or         a6, a6, t2\n\t"
3723        "or         a7, a7, t3\n\t"
3724
3725        "srliw      t0, s1, 8\n\t"
3726        "srliw      t1, s2, 8\n\t"
3727        "srliw      t2, t5, 8\n\t"
3728        "srliw      t3, t6, 8\n\t"
3729        "andi       t0, t0, 0xff\n\t"
3730        "andi       t1, t1, 0xff\n\t"
3731        "andi       t2, t2, 0xff\n\t"
3732        "andi       t3, t3, 0xff\n\t"
3733        "add        t0, t0, %[td4]\n\t"
3734        "add        t1, t1, %[td4]\n\t"
3735        "add        t2, t2, %[td4]\n\t"
3736        "add        t3, t3, %[td4]\n\t"
3737        "lbu        t0, (t0)\n\t"
3738        "lbu        t1, (t1)\n\t"
3739        "lbu        t2, (t2)\n\t"
3740        "lbu        t3, (t3)\n\t"
3741        "slli       t0, t0, 8\n\t"
3742        "slli       t1, t1, 8\n\t"
3743        "slli       t2, t2, 8\n\t"
3744        "slli       t3, t3, 8\n\t"
3745        "or         a4, a4, t0\n\t"
3746        "or         a5, a5, t1\n\t"
3747        "or         a6, a6, t2\n\t"
3748        "or         a7, a7, t3\n\t"
3749
3750        "andi       t0, t6, 0xff\n\t"
3751        "andi       t1, s1, 0xff\n\t"
3752        "andi       t2, s2, 0xff\n\t"
3753        "andi       t3, t5, 0xff\n\t"
3754        "add        t0, t0, %[td4]\n\t"
3755        "add        t1, t1, %[td4]\n\t"
3756        "add        t2, t2, %[td4]\n\t"
3757        "add        t3, t3, %[td4]\n\t"
3758        "lbu        t0, (t0)\n\t"
3759        "lbu        t1, (t1)\n\t"
3760        "lbu        t2, (t2)\n\t"
3761        "lbu        t3, (t3)\n\t"
3762        "or         a4, a4, t0\n\t"
3763        "or         a5, a5, t1\n\t"
3764        "or         a6, a6, t2\n\t"
3765        "or         a7, a7, t3\n\t"
3766
3767        "slli       t4, %[r], 5\n\t"
3768        "add        t4, %[rk], t4\n\t"
3769        "lw         t0,  0(t4)\n\t"
3770        "lw         t1,  4(t4)\n\t"
3771        "lw         t2,  8(t4)\n\t"
3772        "lw         t3, 12(t4)\n\t"
3773        "xor        a4, a4, t0\n\t"
3774        "xor        a5, a5, t1\n\t"
3775        "xor        a6, a6, t2\n\t"
3776        "xor        a7, a7, t3\n\t"
3777
3778        /* Reverse byte in 32-bit words. */
3779#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
3780        STORE_WORD_REV(a4, 0, %[out])
3781        STORE_WORD_REV(a5, 4, %[out])
3782        STORE_WORD_REV(a6, 8, %[out])
3783        STORE_WORD_REV(a7, 12, %[out])
3784#elif !defined(WOLFSSL_RISCV_BIT_MANIPULATION)
3785        "slli        t0, a4, 32\n\t"
3786        "slli        t1, a5, 32\n\t"
3787        "slli        t2, a6, 32\n\t"
3788        "slli        t3, a7, 32\n\t"
3789        "srli        t1, t1, 32\n\t"
3790        "srli        t3, t3, 32\n\t"
3791        "or          t1, t1, t0\n\t"
3792        "or          t3, t3, t2\n\t"
3793        REV8(REG_T1, REG_T1)
3794        REV8(REG_T3, REG_T3)
3795        /* Write encrypted block to output. */
3796        "sd         t1,  0(%[out])\n\t"
3797        "sd         t3,  8(%[out])\n\t"
3798#else
3799        PACK(REG_T1, REG_A5, REG_A4)
3800        PACK(REG_T3, REG_A7, REG_A6)
3801        REV8(REG_T1, REG_T1)
3802        REV8(REG_T3, REG_T3)
3803        /* Write encrypted block to output. */
3804        "sd         t1,  0(%[out])\n\t"
3805        "sd         t3,  8(%[out])\n\t"
3806#endif
3807
3808        :
3809        : [in] "r" (in), [rk] "r" (aes->key), [td] "r" (Td),
3810          [r] "r" (aes->rounds >> 1), [out] "r" (out), [td4] "r" (Td4)
3811        : "memory", "t0", "t1", "t2", "t3", "t4",
3812          "a4", "a5", "a6", "a7",
3813          "t5", "t6", "s1", "s2"
3814    );
3815}
3816#endif /* HAVE_AES_DECRYPT */
3817#endif /* WOLFSSL_AES_DIRECT || HAVE_AES_CBC */
3818
3819#endif /* WOLFSSL_RISCV_SCALAR_CRYPTO_ASM */
3820
3821/* AES-CBC */
3822#if (defined(HAVE_AES_CBC) && !defined(HAVE_AES_CBC_ENC_DEC)) || \
3823    (defined(WOLFSSL_AES_COUNTER) && !defined(HAVE_AES_COUNTER_ENC)) || \
3824    (defined(HAVE_AESGCM) && !defined(WOLFSSL_RISCV_VECTOR_GCM)) || \
3825    defined(HAVE_AESCCM)
3826/* XOR two 16-byte values, out and in, into out.
3827 *
3828 * @param [in, out] out  16-byte value.
3829 * @param [in]      in   16-byte value.
3830 */
3831static WC_INLINE void xorbuf16(byte* out, const byte* in)
3832{
3833    word64* out64 = (word64*)out;
3834    word64* in64  = (word64*)in;
3835
3836    out64[0] ^= in64[0];
3837    out64[1] ^= in64[1];
3838}
3839#endif
3840
3841#if (defined(HAVE_AES_CBC) && !defined(HAVE_AES_CBC_ENC_DEC)) || \
3842    (defined(HAVE_AESGCM) && (!defined(WOLFSSL_RISCV_VECTOR_CRYPTO_ASM) || \
3843     !defined(WOLFSSL_RISCV_VECTOR_GCM)))
3844/* XOR two 16-byte values into out.
3845 *
3846 * @param [out] out  16-byte value.
3847 * @param [in]  a    16-byte value.
3848 * @param [in]  b    16-byte value.
3849 */
3850static WC_INLINE void xorbufout16(byte* out, const byte* a, const byte* b)
3851{
3852    word64* out64 = (word64*)out;
3853    word64* a64   = (word64*)a;
3854    word64* b64   = (word64*)b;
3855
3856    out64[0] = a64[0] ^ b64[0];
3857    out64[1] = a64[1] ^ b64[1];
3858}
3859#endif
3860
3861#if defined(HAVE_AES_CBC) && !defined(HAVE_AES_CBC_ENC_DEC)
3862/* Encrypt blocks of data using AES-CBC.
3863 *
3864 * Implementation using wc_AesEncrypt().
3865 *
3866 * @param [in]  aes  AES object.
3867 * @param [out] out  Encrypted blocks.
3868 * @param [in]  in   Blocks to encrypt.
3869 * @param pin]  sz   Number of bytes to encrypt.
3870 * @return  0 on success.
3871 * @return  BAD_FUNC_ARG when aes, out or in is NULL.
3872 * @return  BAD_LENGTH_E when sz is not a multiple of WC_AES_BLOCK_SIZE.
3873 */
3874int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
3875{
3876    int ret = 0;
3877    word32 blocks = sz / WC_AES_BLOCK_SIZE;
3878
3879    /* Validate parameters. */
3880    if ((aes == NULL) || (out == NULL) || (in == NULL)) {
3881        ret = BAD_FUNC_ARG;
3882    }
3883#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
3884    /* Ensure a multiple of blocks is to be encrypted.  */
3885    if ((ret == 0) && (sz % WC_AES_BLOCK_SIZE)) {
3886        ret = BAD_LENGTH_E;
3887    }
3888#endif
3889
3890    if ((ret == 0) && (blocks > 0)) {
3891        if (in != out) {
3892            /* Encrypt first block with IV. */
3893            xorbufout16(out, (byte*)aes->reg, in);
3894            wc_AesEncrypt(aes, out, out);
3895            in += WC_AES_BLOCK_SIZE;
3896            out += WC_AES_BLOCK_SIZE;
3897            for (blocks--; blocks > 0; blocks--) {
3898                /* Encrypt a block with previous output block as IV. */
3899                xorbufout16(out, out - WC_AES_BLOCK_SIZE, in);
3900                wc_AesEncrypt(aes, out, out);
3901                in += WC_AES_BLOCK_SIZE;
3902                out += WC_AES_BLOCK_SIZE;
3903            }
3904            /* Copy last output block into AES object as next IV. */
3905            memcpy16((byte*)aes->reg, out - WC_AES_BLOCK_SIZE);
3906        }
3907        /* in and out are same buffer. */
3908        else {
3909            byte* data = out;
3910            /* Encrypt first block with IV. */
3911            xorbuf16(data, (byte*)aes->reg);
3912            wc_AesEncrypt(aes, data, data);
3913            data += WC_AES_BLOCK_SIZE;
3914            for (blocks--; blocks > 0; blocks--) {
3915                /* Encrypt a block with previous output block as IV. */
3916                xorbuf16(data, data - WC_AES_BLOCK_SIZE);
3917                wc_AesEncrypt(aes, data, data);
3918                data += WC_AES_BLOCK_SIZE;
3919            }
3920            /* Copy last output block into AES object as next IV. */
3921            memcpy16((byte*)aes->reg, data - WC_AES_BLOCK_SIZE);
3922        }
3923    }
3924
3925    return ret;
3926}
3927
3928#ifdef HAVE_AES_DECRYPT
3929/* Decrypt blocks of data using AES-CBC.
3930 *
3931 * Implementation using wc_AesDecrypt().
3932 *
3933 * @param [in]  aes  AES object.
3934 * @param [out] out  Decrypted blocks.
3935 * @param [in]  in   Blocks to decrypt.
3936 * @param pin]  sz   Number of bytes to decrypt.
3937 * @return  0 on success.
3938 * @return  BAD_FUNC_ARG when aes, out or in is NULL.
3939 * @return  BAD_FUNC_ARG when sz is not a multiple of WC_AES_BLOCK_SIZE.
3940 * @return  BAD_LENGTH_E when sz is not a multiple of WC_AES_BLOCK_SIZE.
3941 */
3942int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
3943{
3944    int ret = 0;
3945    word32 blocks = sz / WC_AES_BLOCK_SIZE;
3946
3947    /* Validate parameters. */
3948    if ((aes == NULL) || (out == NULL) || (in == NULL)) {
3949        ret = BAD_FUNC_ARG;
3950    }
3951    /* Ensure a multiple of blocks is being decrypted.  */
3952    if ((ret == 0) && (sz % WC_AES_BLOCK_SIZE)) {
3953#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
3954        ret = BAD_LENGTH_E;
3955#else
3956        ret = BAD_FUNC_ARG;
3957#endif
3958    }
3959
3960    if ((ret == 0) && (blocks > 0)) {
3961        if (in != out) {
3962            /* Decrypt first block with the IV. */
3963            wc_AesDecrypt(aes, in, out);
3964            xorbuf16(out, (byte*)aes->reg);
3965            in += WC_AES_BLOCK_SIZE;
3966            out += WC_AES_BLOCK_SIZE;
3967            for (blocks--; blocks > 0; blocks--) {
3968                /* Decrypt a block with previous input block as IV. */
3969                wc_AesDecrypt(aes, in, out);
3970                xorbuf16(out, in - WC_AES_BLOCK_SIZE);
3971                in += WC_AES_BLOCK_SIZE;
3972                out += WC_AES_BLOCK_SIZE;
3973            }
3974            /* Copy last output block into AES object as next IV. */
3975            memcpy16((byte*)aes->reg, in - WC_AES_BLOCK_SIZE);
3976        }
3977        /* in and out are same buffer. */
3978        else {
3979            byte* data = out;
3980            for (; blocks > 0; blocks -= 2) {
3981                /* Decrypt block with the IV in aes->reg. */
3982                memcpy16((byte*)aes->tmp, data);
3983                wc_AesDecrypt(aes, data, data);
3984                xorbuf16(data, (byte*)aes->reg);
3985                if (blocks == 1) {
3986                    memcpy16((byte*)aes->reg, (byte*)aes->tmp);
3987                    break;
3988                }
3989                data += WC_AES_BLOCK_SIZE;
3990                /* Decrypt block with the IV in aes->tmp. */
3991                memcpy16((byte*)aes->reg, data);
3992                wc_AesDecrypt(aes, data, data);
3993                xorbuf16(data, (byte*)aes->tmp);
3994                data += WC_AES_BLOCK_SIZE;
3995            }
3996        }
3997    }
3998
3999    return ret;
4000}
4001#endif
4002#endif
4003
4004/* AES-ECB */
4005#ifdef HAVE_AES_ECB
4006/* Encrypt blocks of data using AES-ECB.
4007 *
4008 * @param [in]  aes  AES object.
4009 * @param [out] out  Encrypted blocks.
4010 * @param [in]  in   Blocks to encrypt.
4011 * @param pin]  sz   Number of bytes to encrypt.
4012 * @return  0 on success.
4013 * @return  BAD_FUNC_ARG when aes, out or in is NULL.
4014 * @return  BAD_LENGTH_E when sz is not a multiple of WC_AES_BLOCK_SIZE.
4015 */
4016int wc_AesEcbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
4017{
4018    int ret = 0;
4019
4020    /* Validate parameters. */
4021    if ((aes == NULL) || (out == NULL) || (in == NULL)) {
4022        ret = BAD_FUNC_ARG;
4023    }
4024    /* Ensure a multiple of blocks is to be encrypted.  */
4025    if ((ret == 0) && ((sz % WC_AES_BLOCK_SIZE) != 0)) {
4026        ret = BAD_LENGTH_E;
4027    }
4028
4029    if (ret == 0) {
4030        /* Encrypt block by block. */
4031        while (sz > 0) {
4032            wc_AesEncrypt(aes, in, out);
4033            out += WC_AES_BLOCK_SIZE;
4034            in += WC_AES_BLOCK_SIZE;
4035            sz -= WC_AES_BLOCK_SIZE;
4036        }
4037    }
4038
4039    return ret;
4040}
4041
4042#ifdef HAVE_AES_DECRYPT
4043/* Decrypt blocks of data using AES-ECB.
4044 *
4045 * @param [in]  aes  AES object.
4046 * @param [out] out  Encrypted blocks.
4047 * @param [in]  in   Blocks to encrypt.
4048 * @param pin]  sz   Number of bytes to encrypt.
4049 * @return  0 on success.
4050 * @return  BAD_FUNC_ARG when aes, out or in is NULL.
4051 * @return  BAD_LENGTH_E when sz is not a multiple of WC_AES_BLOCK_SIZE.
4052 */
4053int wc_AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
4054{
4055    int ret = 0;
4056
4057    /* Validate parameters. */
4058    if ((aes == NULL) || (out == NULL) || (in == NULL)) {
4059        ret = BAD_FUNC_ARG;
4060    }
4061    /* Ensure a multiple of blocks is to be decrypted.  */
4062    if ((ret == 0) && ((sz % WC_AES_BLOCK_SIZE) != 0)) {
4063        ret = BAD_LENGTH_E;
4064    }
4065
4066    if (ret == 0) {
4067        /* Decrypt block by block. */
4068        while (sz > 0) {
4069            wc_AesDecrypt(aes, in, out);
4070            out += WC_AES_BLOCK_SIZE;
4071            in += WC_AES_BLOCK_SIZE;
4072            sz -= WC_AES_BLOCK_SIZE;
4073        }
4074    }
4075
4076    return ret;
4077}
4078#endif
4079#endif /* HAVE_AES_ECB */
4080
4081/* AES-CTR */
4082#if defined(WOLFSSL_AES_COUNTER) && !defined(HAVE_AES_COUNTER_ENC)
4083/* Increment AES counter.
4084 *
4085 * Big-endian byte ordering.
4086 *
4087 * @param [in, out] inOutCtr  Counter value to be incremented.
4088 */
4089static WC_INLINE void IncrementAesCounter(byte* inOutCtr)
4090{
4091    int i;
4092
4093    /* Big-endian array - start at last element and move back. */
4094    for (i = WC_AES_BLOCK_SIZE - 1; i >= 0; i--) {
4095        /* Result not zero means no carry. */
4096        if ((++inOutCtr[i]) != 0) {
4097            return;
4098        }
4099    }
4100}
4101
4102/* Encrypt blocks of data using AES-CTR.
4103 *
4104 * Implementation uses wc_AesEncrypt().
4105 *
4106 * @param [in]  aes  AES object.
4107 * @param [out] out  Encrypted blocks.
4108 * @param [in]  in   Blocks to encrypt.
4109 * @param [in]  sz   Number of bytes to encrypt.
4110 * @return  0 on success.
4111 * @return  BAD_FUNC_ARG when aes, out or in is NULL.
4112 * @return  BAD_FUNC_ARG when key size in AES object is not supported.
4113 */
4114int wc_AesCtrEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
4115{
4116    byte scratch[WC_AES_BLOCK_SIZE];
4117    word32 processed;
4118    int ret = 0;
4119
4120    /* Validate parameters. */
4121    if (aes == NULL || out == NULL || in == NULL) {
4122        ret = BAD_FUNC_ARG;
4123    }
4124    if (ret == 0) {
4125        /* Check key size is supported. */
4126        switch(aes->rounds) {
4127        #ifdef WOLFSSL_AES_128
4128            case 10: /* AES 128 BLOCK */
4129        #endif /* WOLFSSL_AES_128 */
4130        #ifdef WOLFSSL_AES_192
4131            case 12: /* AES 192 BLOCK */
4132        #endif /* WOLFSSL_AES_192 */
4133        #ifdef WOLFSSL_AES_256
4134            case 14: /* AES 256 BLOCK */
4135        #endif /* WOLFSSL_AES_256 */
4136                break;
4137            default:
4138                WOLFSSL_MSG("Bad AES-CTR round value");
4139                ret = BAD_FUNC_ARG;
4140        }
4141    }
4142
4143    if (ret == 0) {
4144        /* Use up any unused bytes left in aes->tmp */
4145        processed = min(aes->left, sz);
4146        if (processed > 0) {
4147            /* XOR in encrypted counter.  */
4148            xorbufout(out, in, (byte*)aes->tmp + WC_AES_BLOCK_SIZE - aes->left,
4149                processed);
4150            out += processed;
4151            in += processed;
4152            aes->left -= processed;
4153            sz -= processed;
4154        }
4155
4156        /* Do whole blocks of data. */
4157        while (sz >= WC_AES_BLOCK_SIZE) {
4158            wc_AesEncrypt(aes, (byte*)aes->reg, scratch);
4159            xorbuf16(scratch, in);
4160            memcpy16(out, scratch);
4161            IncrementAesCounter((byte*)aes->reg);
4162
4163            out += WC_AES_BLOCK_SIZE;
4164            in  += WC_AES_BLOCK_SIZE;
4165            sz  -= WC_AES_BLOCK_SIZE;
4166            aes->left = 0;
4167        }
4168        ForceZero(scratch, WC_AES_BLOCK_SIZE);
4169
4170        if (sz > 0) {
4171            /* Encrypt counter and store in aes->tmp.
4172             * Use up aes->tmp to encrypt data less than a block.
4173             */
4174            wc_AesEncrypt(aes, (byte*)aes->reg, (byte*)aes->tmp);
4175            IncrementAesCounter((byte*)aes->reg);
4176            aes->left = WC_AES_BLOCK_SIZE - sz;
4177            /* XOR in encrypted counter. */
4178            xorbufout(out, in, aes->tmp, sz);
4179        }
4180    }
4181
4182    return ret;
4183}
4184
4185#endif /* WOLFSSL_AES_COUNTER */
4186
4187#if defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER)
4188/* Set AES key directly.
4189 *
4190 * @param [in] aes     AES object.
4191 * @param [in] key     Secret key to use.
4192 * @param [in] keyLen  Length of key in bytes.
4193 * @param [in] iv      Initialization Vector (IV) to use. May be NULL.
4194 * @param [in] dir     Direction of crypt: AES_ENCRYPT, AES_DECRYPT.
4195 * @return  0 on success.
4196 * @return  BAD_FUNC_ARG when aes or key is NULL.
4197 * @return  BAD_FUNC_ARG when keyLen/dir is not supported or valid.
4198 */
4199int wc_AesSetKeyDirect(Aes* aes, const byte* key, word32 keyLen, const byte* iv,
4200    int dir)
4201{
4202    return wc_AesSetKey(aes, key, keyLen, iv, dir);
4203}
4204#endif
4205
4206/* Set the IV.
4207 *
4208 * @param [in] aes  AES object.
4209 * @param [in] iv   Initialization Vector (IV) to set.
4210 *                  When NULL, an IV of all zeros is set.
4211 * @return  0 on success.
4212 * @return  BAD_FUNC_ARG when aes is NULL.
4213 */
4214int wc_AesSetIV(Aes* aes, const byte* iv)
4215{
4216    int ret = 0;
4217
4218    if (aes == NULL) {
4219        ret = BAD_FUNC_ARG;
4220    }
4221    else if (iv != NULL) {
4222        memcpy16((byte*)aes->reg, iv);
4223    }
4224    else {
4225        XMEMSET(aes->reg,  0, WC_AES_BLOCK_SIZE);
4226    }
4227
4228    return ret;
4229}
4230
4231/* AES-DIRECT */
4232#ifdef WOLFSSL_AES_DIRECT
4233/* Direct encryption of a block.
4234 *
4235 * @param [in]  aes  AES object.
4236 * @param [out] out  Encrypted block.
4237 * @param [in]  in   Block to encrypt.
4238 * @return  0 on success.
4239 * @return  BAD_FUNC_ARG when aes, out, or in is NULL.
4240 */
4241int wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in)
4242{
4243    int ret = 0;
4244
4245    if ((aes == NULL) || (out == NULL) || (in == NULL)) {
4246        WOLFSSL_MSG("Invalid input to wc_AesEncryptDirect");
4247        ret = BAD_FUNC_ARG;
4248    }
4249    if (ret == 0) {
4250        wc_AesEncrypt(aes, in, out);
4251    }
4252
4253    return ret;
4254}
4255#ifdef HAVE_AES_DECRYPT
4256/* Direct decryption of a block.
4257 *
4258 * @param [in]  aes  AES object.
4259 * @param [out] out  Decrypted block.
4260 * @param [in]  in   Block to decrypt.
4261 * @return  0 on success.
4262 * @return  BAD_FUNC_ARG when aes, out, or in is NULL.
4263 */
4264int wc_AesDecryptDirect(Aes* aes, byte* out, const byte* in)
4265{
4266    int ret = 0;
4267
4268    if ((aes == NULL) || (out == NULL) || (in == NULL)) {
4269        WOLFSSL_MSG("Invalid input to wc_AesDecryptDirect");
4270        ret = BAD_FUNC_ARG;
4271    }
4272    if (ret == 0) {
4273        wc_AesDecrypt(aes, in, out);
4274    }
4275
4276    return ret;
4277}
4278#endif /* HAVE_AES_DECRYPT */
4279#endif /* WOLFSSL_AES_DIRECT */
4280
4281#ifdef WOLFSSL_AES_COUNTER
4282
4283/* Set the key for AES-CTR.
4284 *
4285 * @param [in] aes     AES object.
4286 * @param [in] key     Secret key to use.
4287 * @param [in] keyLen  Length of key in bytes.
4288 * @param [in] iv      Initialization Vector (IV) to use. May be NULL.
4289 * @param [in] dir     Direction of crypt: AES_ENCRYPT, AES_DECRYPT.
4290 *                     For CTR mode, underlying key is always for encryption.
4291 * @return  0 on success.
4292 * @return  BAD_FUNC_ARG when aes or key is NULL.
4293 * @return  BAD_FUNC_ARG when keyLen is not supported or valid.
4294 */
4295int wc_AesCtrSetKey(Aes* aes, const byte* key, word32 len, const byte* iv,
4296    int dir)
4297{
4298    (void)dir;
4299    return wc_AesSetKey(aes, key, len, iv, AES_ENCRYPTION);
4300}
4301
4302#endif /* WOLFSSL_AES_COUNTER */
4303
4304#ifdef HAVE_AESGCM
4305
4306#if !defined(WOLFSSL_RISCV_VECTOR_GCM) && \
4307    !defined(WOLFSSL_RISCV_VECTOR_CARRYLESS) && \
4308    !defined(WOLFSSL_RISCV_CARRYLESS)
4309/* Shift x in GF2
4310 *
4311 * @param [in, out] x  128-bit value to shift.
4312 */
4313static WC_INLINE void RIGHTSHIFTX(byte* x)
4314{
4315    int i;
4316    int carryIn = 0;
4317    byte borrow = (0x00 - (x[15] & 0x01)) & 0xE1;
4318
4319    for (i = 0; i < WC_AES_BLOCK_SIZE; i++) {
4320        int carryOut = (x[i] & 0x01) << 7;
4321        x[i] = (byte) ((x[i] >> 1) | carryIn);
4322        carryIn = carryOut;
4323    }
4324    x[0] ^= borrow;
4325}
4326
4327/* Shift right by 4 a big-endian value in little-endian.
4328 *
4329 * @param [out] r8  Result of shift.
4330 * @param [in]  z8  128-bit value to shift.
4331 */
4332static WC_INLINE void Shift4_M0(byte *r8, byte *z8)
4333{
4334    int i;
4335    for (i = 15; i > 0; i--)
4336        r8[i] = (byte)(z8[i-1] << 4) | (byte)(z8[i] >> 4);
4337    r8[0] = (byte)(z8[0] >> 4);
4338}
4339
4340/* Generate 4-bit table.
4341 *
4342 * @param [in, out] gcm  GCM object.
4343 */
4344void GenerateM0(Gcm* gcm)
4345{
4346    int i;
4347    byte (*m)[WC_AES_BLOCK_SIZE] = gcm->M0;
4348
4349    /* 0 times -> 0x0 */
4350    XMEMSET(m[0x0], 0, WC_AES_BLOCK_SIZE);
4351    /* 1 times -> 0x8 */
4352    memcpy16(m[0x8], gcm->H);
4353    /* 2 times -> 0x4 */
4354    memcpy16(m[0x4], m[0x8]);
4355    RIGHTSHIFTX(m[0x4]);
4356    /* 4 times -> 0x2 */
4357    memcpy16(m[0x2], m[0x4]);
4358    RIGHTSHIFTX(m[0x2]);
4359    /* 8 times -> 0x1 */
4360    memcpy16(m[0x1], m[0x2]);
4361    RIGHTSHIFTX(m[0x1]);
4362
4363    /* 0x3 */
4364    memcpy16(m[0x3], m[0x2]);
4365    xorbuf16(m[0x3], m[0x1]);
4366
4367    /* 0x5 -> 0x7 */
4368    memcpy16(m[0x5], m[0x4]);
4369    xorbuf16(m[0x5], m[0x1]);
4370    memcpy16(m[0x6], m[0x4]);
4371    xorbuf16(m[0x6], m[0x2]);
4372    memcpy16(m[0x7], m[0x4]);
4373    xorbuf16(m[0x7], m[0x3]);
4374
4375    /* 0x9 -> 0xf */
4376    memcpy16(m[0x9], m[0x8]);
4377    xorbuf16(m[0x9], m[0x1]);
4378    memcpy16(m[0xa], m[0x8]);
4379    xorbuf16(m[0xa], m[0x2]);
4380    memcpy16(m[0xb], m[0x8]);
4381    xorbuf16(m[0xb], m[0x3]);
4382    memcpy16(m[0xc], m[0x8]);
4383    xorbuf16(m[0xc], m[0x4]);
4384    memcpy16(m[0xd], m[0x8]);
4385    xorbuf16(m[0xd], m[0x5]);
4386    memcpy16(m[0xe], m[0x8]);
4387    xorbuf16(m[0xe], m[0x6]);
4388    memcpy16(m[0xf], m[0x8]);
4389    xorbuf16(m[0xf], m[0x7]);
4390
4391    for (i = 0; i < 16; i++) {
4392        Shift4_M0(m[16+i], m[i]);
4393    }
4394}
4395#endif
4396
4397/* Setup the AES-GCM operation with the key.
4398 *
4399 * @param [in] aes  AES object.
4400 * @param [in] key  Secret key to use.
4401 * @param [in] ken  Length of key in bytes.
4402 * @return  0 on success.
4403 * @return  BAD_FUNC_ARG when aes or key is NULL.
4404 * @return  BAD_FUNC_ARG when the key length is not supported.
4405 */
4406int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len)
4407{
4408    int  ret = 0;
4409    byte iv[WC_AES_BLOCK_SIZE];
4410
4411    if (aes == NULL) {
4412        ret = BAD_FUNC_ARG;
4413    }
4414    if ((ret == 0) && (len != 16) && (len != 24) && (len != 32)) {
4415        ret = BAD_FUNC_ARG;
4416    }
4417
4418    if (ret == 0) {
4419        XMEMSET(iv, 0, WC_AES_BLOCK_SIZE);
4420        ret = wc_AesSetKey(aes, key, len, iv, AES_ENCRYPTION);
4421    }
4422    if (ret == 0) {
4423        wc_AesEncrypt(aes, (byte*)aes->reg, aes->gcm.H);
4424#ifdef WOLFSSL_RISCV_VECTOR_GCM
4425        /* Vector crypto instructions do bit reversal. */
4426#elif defined(WOLFSSL_RISCV_VECTOR_CARRYLESS)
4427        /* Vector crypto instructions do bit reversal. */
4428#elif defined(WOLFSSL_RISCV_CARRYLESS)
4429        /* Reverse bits in aes->gcm.H. */
4430#ifdef WOLFSSL_RISCV_BIT_MANIPULATION
4431        __asm__ __volatile__ (
4432            "ld          t0, 0(%[data])\n\t"
4433            "ld          t1, 8(%[data])\n\t"
4434            BREV8(REG_T0, REG_T0)
4435            BREV8(REG_T1, REG_T1)
4436            "sd          t0, 0(%[data])\n\t"
4437            "sd          t1, 8(%[data])\n\t"
4438        :
4439        : [data] "r" (aes->gcm.H)
4440        : "memory", "t0", "t1"
4441        );
4442#else
4443        __asm__ __volatile__ (
4444            "ld          t0, 0(%[data])\n\t"
4445            "ld          t1, 8(%[data])\n\t"
4446
4447            /* Swap odd-even bits. */
4448            "li          t4, 0x5555555555555555\n\t"
4449            "srli        t2, t0, 1\n\t"
4450            "srli        t3, t1, 1\n\t"
4451            "and         t0, t0, t4\n\t"
4452            "and         t1, t1, t4\n\t"
4453            "and         t2, t2, t4\n\t"
4454            "and         t3, t3, t4\n\t"
4455            "slli        t0, t0, 1\n\t"
4456            "slli        t1, t1, 1\n\t"
4457            "or          t0, t0, t2\n\t"
4458            "or          t1, t1, t3\n\t"
4459            /* Swap pairs. */
4460            "li          t4, 0x3333333333333333\n\t"
4461            "srli        t2, t0, 2\n\t"
4462            "srli        t3, t1, 2\n\t"
4463            "and         t0, t0, t4\n\t"
4464            "and         t1, t1, t4\n\t"
4465            "and         t2, t2, t4\n\t"
4466            "and         t3, t3, t4\n\t"
4467            "slli        t0, t0, 2\n\t"
4468            "slli        t1, t1, 2\n\t"
4469            "or          t0, t0, t2\n\t"
4470            "or          t1, t1, t3\n\t"
4471            /* Swap nibbles. */
4472            "li          t4, 0x0f0f0f0f0f0f0f0f\n\t"
4473            "srli        t2, t0, 4\n\t"
4474            "srli        t3, t1, 4\n\t"
4475            "and         t0, t0, t4\n\t"
4476            "and         t1, t1, t4\n\t"
4477            "and         t2, t2, t4\n\t"
4478            "and         t3, t3, t4\n\t"
4479            "slli        t0, t0, 4\n\t"
4480            "slli        t1, t1, 4\n\t"
4481            "or          t0, t0, t2\n\t"
4482            "or          t1, t1, t3\n\t"
4483
4484            "sd          t0, 0(%[data])\n\t"
4485            "sd          t1, 8(%[data])\n\t"
4486        :
4487        : [data] "r" (aes->gcm.H)
4488        : "memory", "t0", "t1", "t2", "t3", "t4"
4489        );
4490#endif /* WOLFSSL_RISCV_BIT_MANIPULATION */
4491#else
4492        GenerateM0(&aes->gcm);
4493#endif
4494    }
4495
4496    return ret;
4497}
4498
4499#ifndef WOLFSSL_RISCV_VECTOR_GCM
4500/* Encode sz in bytes into array as big-endian number of bits.
4501 *
4502 * @param [out] buf  Buffer to encode size into.
4503 * @param [in]  sz   Size in bytes.
4504 */
4505static WC_INLINE void FlattenSzInBits(byte* buf, word32 sz)
4506{
4507#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
4508    __asm__ __volatile__ (
4509        /* sz is only 32-bits */
4510        /* Multiply by 8 to get size in bits. */
4511        "slli       %[sz], %[sz], 3\n\t"
4512        "srli       t0, %[sz], 32\n\t"
4513        "srli       t1, %[sz], 24\n\t"
4514        "srli       t2, %[sz], 16\n\t"
4515        "srli       t3, %[sz], 8\n\t"
4516        /* Top 3 bytes are 0. */
4517        "sh         x0   , 0(%[buf])\n\t"
4518        "sb         x0   , 2(%[buf])\n\t"
4519        "sb         t0   , 3(%[buf])\n\t"
4520        "sb         t1   , 4(%[buf])\n\t"
4521        "sb         t2   , 5(%[buf])\n\t"
4522        "sb         t3   , 6(%[buf])\n\t"
4523        "sb         %[sz], 7(%[buf])\n\t"
4524        : [sz] "+r" (sz)
4525        : [buf] "r" (buf)
4526        : "memory", "t0", "t1", "t2", "t3"
4527    );
4528#else
4529    __asm__ __volatile__ (
4530        "slli       t0, %[sz], 3\n\t"
4531        REV8(REG_T0, REG_T0)
4532        "sd         t0, 0(%[buf])\n\t"
4533        :
4534        : [sz] "r" (sz), [buf] "r" (buf)
4535        : "memory", "t0"
4536    );
4537#endif
4538}
4539#endif
4540
4541#if defined(WOLFSSL_RISCV_VECTOR_GCM)
4542
4543/* Vector GHASH: vd = (vd ^ vs1) * vs2 */
4544#define VGHSH_VV(vd, vs1, vs2)                                              \
4545    ASM_WORD((0b101100 << 26) | (0b1 << 25) | (0b010 << 12) |               \
4546             (0b1110111 << 0) | (vs2 << 20) | (vs1 << 15) | (vd << 7))
4547/* Vector GMULT: vd = vd * vs2 */
4548#define VGMUL_VV(vd, vs2)                                                   \
4549    ASM_WORD((0b101000 << 26) | (0b1 << 25) | (0b010 << 12) |               \
4550             (0b1110111 << 0) | (vs2 << 20) | (0b10001 << 15) | (vd << 7))
4551
4552/* GHASH Additional Authentication Data (AAD) and cipher text.
4553 *
4554 * @param [in]  gcm  GCM object.
4555 * @param [in]  a    Additional Authentication Data (AAD).
4556 * @param [in]  aSz  Size of AAD in bytes.
4557 * @param [in]  c    Cipher text.
4558 * @param [in]  cSz  Size of cipher text in bytes.
4559 * @param [out] s    Hash result.
4560 * @param [in]  sSz  Number of bytes to put into hash result.
4561 */
4562void GHASH(Gcm* gcm, const byte* a, word32 aSz, const byte* c, word32 cSz,
4563    byte* s, word32 sSz)
4564{
4565    if (gcm != NULL) {
4566        byte x[WC_AES_BLOCK_SIZE];
4567        byte scratch[WC_AES_BLOCK_SIZE];
4568        byte* h = gcm->H;
4569
4570        __asm__ __volatile__ (
4571            VXOR_VV(REG_V0, REG_V0, REG_V0)
4572
4573            /* Hash in A, the Additional Authentication Data */
4574            "beqz       %[aSz], L_ghash_aad_done\n\t"
4575            "beqz       %[a], L_ghash_aad_done\n\t"
4576
4577            "srli       t3, %[aSz], 4\n\t"
4578            VSETIVLI(REG_T0, 4, 0, 0, 0b010, 0b000)
4579            "mv         t0, %[h]\n\t"
4580            VL1RE32_V(REG_V1, REG_T0)
4581
4582            "beqz       t3, L_ghash_aad_blocks_done\n\t"
4583         "L_ghash_aad_loop:\n\t"
4584            "mv         t0, %[a]\n\t"
4585            VL1RE32_V(REG_V2, REG_T0)
4586            VGHSH_VV(REG_V0, REG_V2, REG_V1)
4587            "addi       %[a], %[a], 16\n\t"
4588            "addi       t3, t3, -1\n\t"
4589            "bnez       t3, L_ghash_aad_loop\n\t"
4590         "L_ghash_aad_blocks_done:\n\t"
4591            "andi       t3, %[aSz], 0xf\n\t"
4592            "beqz       t3, L_ghash_aad_done\n\t"
4593            VXOR_VV(REG_V2, REG_V2, REG_V2)
4594            "mv         t0, %[scratch]\n\t"
4595            VS1R_V(REG_V2, REG_T0)
4596            "mv         t2, t3\n\t"
4597         "L_ghash_aad_load_byte:\n\t"
4598            "lb         t0, (%[a])\n\t"
4599            "sb         t0, (%[scratch])\n\t"
4600            "addi       %[a], %[a], 1\n\t"
4601            "addi       %[scratch], %[scratch], 1\n\t"
4602            "addi       t2, t2, -1\n\t"
4603            "bnez       t2, L_ghash_aad_load_byte\n\t"
4604            "sub        %[scratch], %[scratch], t3\n\t"
4605            "mv         t0, %[scratch]\n\t"
4606            VL1RE32_V(REG_V2, REG_T0)
4607            VGHSH_VV(REG_V0, REG_V2, REG_V1)
4608         "L_ghash_aad_done:\n\t"
4609
4610            /* Hash in C, the Ciphertext */
4611            "beqz       %[cSz], L_ghash_ct_done\n\t"
4612            "beqz       %[c], L_ghash_ct_done\n\t"
4613
4614            "srli       t3, %[cSz], 4\n\t"
4615            VSETIVLI(REG_T0, 4, 0, 0, 0b010, 0b000)
4616            "mv         t0, %[h]\n\t"
4617            VL1RE32_V(REG_V1, REG_T0)
4618
4619            "beqz       t3, L_ghash_ct_blocks_done\n\t"
4620         "L_ghash_ct_loop:\n\t"
4621            "mv         t0, %[c]\n\t"
4622            VL1RE32_V(REG_V2, REG_T0)
4623            VGHSH_VV(REG_V0, REG_V2, REG_V1)
4624            "addi       %[c], %[c], 16\n\t"
4625            "addi       t3, t3, -1\n\t"
4626            "bnez       t3, L_ghash_ct_loop\n\t"
4627         "L_ghash_ct_blocks_done:\n\t"
4628            "andi       t3, %[cSz], 0xf\n\t"
4629            "beqz       t3, L_ghash_ct_done\n\t"
4630            VXOR_VV(REG_V2, REG_V2, REG_V2)
4631            "mv         t0, %[scratch]\n\t"
4632            VS1R_V(REG_V2, REG_T0)
4633            "mv         t2, t3\n\t"
4634         "L_ghash_ct_load_byte:\n\t"
4635            "lb         t0, (%[c])\n\t"
4636            "sb         t0, (%[scratch])\n\t"
4637            "addi       %[c], %[c], 1\n\t"
4638            "addi       %[scratch], %[scratch], 1\n\t"
4639            "addi       t2, t2, -1\n\t"
4640            "bnez       t2, L_ghash_ct_load_byte\n\t"
4641            "sub        %[scratch], %[scratch], t3\n\t"
4642            "mv         t0, %[scratch]\n\t"
4643            VL1RE32_V(REG_V2, REG_T0)
4644            VGHSH_VV(REG_V0, REG_V2, REG_V1)
4645         "L_ghash_ct_done:\n\t"
4646
4647            /* Hash in the lengths of A and C in bits */
4648        #ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
4649            /* aSz is only 32-bits */
4650            /* Multiply by 8 do get size in bits. */
4651            "slli       %[aSz], %[aSz], 3\n\t"
4652            "srli       t0, %[aSz], 32\n\t"
4653            "srli       t1, %[aSz], 24\n\t"
4654            "srli       t2, %[aSz], 16\n\t"
4655            "srli       t3, %[aSz], 8\n\t"
4656            /* Top 3 bytes are 0. */
4657            "sh         x0    , 0(%[scratch])\n\t"
4658            "sb         x0    , 2(%[scratch])\n\t"
4659            "sb         t0    , 3(%[scratch])\n\t"
4660            "sb         t1    , 4(%[scratch])\n\t"
4661            "sb         t2    , 5(%[scratch])\n\t"
4662            "sb         t3    , 6(%[scratch])\n\t"
4663            "sb         %[aSz], 7(%[scratch])\n\t"
4664            /* cSz is only 32-bits */
4665            /* Multiply by 8 do get size in bits. */
4666            "slli       %[cSz], %[cSz], 3\n\t"
4667            "srli       t0, %[cSz], 32\n\t"
4668            "srli       t1, %[cSz], 24\n\t"
4669            "srli       t2, %[cSz], 16\n\t"
4670            "srli       t3, %[cSz], 8\n\t"
4671            /* Top 3 bytes are 0. */
4672            "sh         x0    ,  8(%[scratch])\n\t"
4673            "sb         x0    , 10(%[scratch])\n\t"
4674            "sb         t0    , 11(%[scratch])\n\t"
4675            "sb         t1    , 12(%[scratch])\n\t"
4676            "sb         t2    , 13(%[scratch])\n\t"
4677            "sb         t3    , 14(%[scratch])\n\t"
4678            "sb         %[cSz], 15(%[scratch])\n\t"
4679        #else
4680            "slli       t0, %[aSz], 3\n\t"
4681            REV8(REG_T0, REG_T0)
4682            "sd         t0, 0(%[scratch])\n\t"
4683            "slli       t0, %[cSz], 3\n\t"
4684            REV8(REG_T0, REG_T0)
4685            "sd         t0, 8(%[scratch])\n\t"
4686        #endif
4687            "mv         t0, %[scratch]\n\t"
4688            VL1RE32_V(REG_V2, REG_T0)
4689            VGHSH_VV(REG_V0, REG_V2, REG_V1)
4690
4691            "mv         t1, %[x]\n\t"
4692            VS1R_V(REG_V0, REG_T1)
4693
4694            : [a] "+r" (a), [c] "+r" (c) , [aSz] "+r" (aSz), [cSz] "+r" (cSz)
4695            : [x] "r" (x), [h] "r" (h), [scratch] "r" (scratch)
4696            : "memory", "t0", "t1", "t2", "t3"
4697        );
4698
4699        /* Copy the result into s. */
4700        XMEMCPY(s, x, sSz);
4701    }
4702}
4703
4704#define HAVE_GHASH
4705
4706#elif defined(WOLFSSL_RISCV_VECTOR_CARRYLESS)
4707
4708#define VCLMUL_VV(vd, vs1, vs2) \
4709    ASM_WORD((0b001100 << 26) | (0b1 << 25) | (0b010 << 12) | \
4710             (0b1010111 << 0) | (vs2 << 20) | (vs1 << 15) | (vd << 7))
4711#define VCLMULH_VV(vd, vs1, vs2) \
4712    ASM_WORD((0b001101 << 26) | (0b1 << 25) | (0b010 << 12) | \
4713             (0b1010111 << 0) | (vs2 << 20) | (vs1 << 15) | (vd << 7))
4714
4715/* GMULT, multiply in GF2, x and y into x.
4716 *
4717 * @param [in, out]  x  On in, value to GMULT.
4718 *                      On out, result of GMULT.
4719 * @param [in]       y  Value to GMULT.
4720 */
4721static void GMULT(byte* x, byte* y)
4722{
4723    static byte red[16] = {
4724        0x87, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
4725        0x87, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
4726    };
4727    __asm__ __volatile__ (
4728        VSETIVLI(REG_X0, 2, 0, 0, 0b011, 0b000)
4729
4730        /* 0x87 into both 64-bit elements of v7. */
4731        "mv          t1, %[red]\n\t"
4732        VL1RE64_V(REG_V8, REG_T1)
4733
4734        "mv          t1, %[x]\n\t"
4735        VL1RE64_V(REG_V0, REG_T1)
4736        "mv          t0, %[y]\n\t"
4737        VL1RE64_V(REG_V1, REG_T0)
4738        /* Reverse x and y. */
4739#ifdef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
4740        VBREV8(REG_V0, REG_V0)
4741        VBREV8(REG_V1, REG_V1)
4742#else
4743        VSETIVLI(REG_X0, 16, 0, 0, 0b000, 0b000)
4744
4745        /* Swap odd/even bits. */
4746        "li          t0, 0x55\n\t"
4747        VMV_V_X(REG_V4, REG_T0)
4748        VSRL_VI(REG_V2, REG_V0, 1)
4749        VSRL_VI(REG_V3, REG_V1, 1)
4750        VAND_VV(REG_V0, REG_V0, REG_V4)
4751        VAND_VV(REG_V1, REG_V1, REG_V4)
4752        VAND_VV(REG_V2, REG_V2, REG_V4)
4753        VAND_VV(REG_V3, REG_V3, REG_V4)
4754        VSLL_VI(REG_V0, REG_V0, 1)
4755        VSLL_VI(REG_V1, REG_V1, 1)
4756        VOR_VV(REG_V0, REG_V0, REG_V2)
4757        VOR_VV(REG_V1, REG_V1, REG_V3)
4758        /* Swap pairs of bits. */
4759        "li          t0, 0x33\n\t"
4760        VMV_V_X(REG_V4, REG_T0)
4761        VSRL_VI(REG_V2, REG_V0, 2)
4762        VSRL_VI(REG_V3, REG_V1, 2)
4763        VAND_VV(REG_V0, REG_V0, REG_V4)
4764        VAND_VV(REG_V1, REG_V1, REG_V4)
4765        VAND_VV(REG_V2, REG_V2, REG_V4)
4766        VAND_VV(REG_V3, REG_V3, REG_V4)
4767        VSLL_VI(REG_V0, REG_V0, 2)
4768        VSLL_VI(REG_V1, REG_V1, 2)
4769        VOR_VV(REG_V0, REG_V0, REG_V2)
4770        VOR_VV(REG_V1, REG_V1, REG_V3)
4771        /* Swap nibbles. */
4772        "li          t0, 0x0f\n\t"
4773        VMV_V_X(REG_V4, REG_T0)
4774        VSRL_VI(REG_V2, REG_V0, 4)
4775        VSRL_VI(REG_V3, REG_V1, 4)
4776        VAND_VV(REG_V0, REG_V0, REG_V4)
4777        VAND_VV(REG_V1, REG_V1, REG_V4)
4778        VAND_VV(REG_V2, REG_V2, REG_V4)
4779        VAND_VV(REG_V3, REG_V3, REG_V4)
4780        VSLL_VI(REG_V0, REG_V0, 4)
4781        VSLL_VI(REG_V1, REG_V1, 4)
4782        VOR_VV(REG_V0, REG_V0, REG_V2)
4783        VOR_VV(REG_V1, REG_V1, REG_V3)
4784
4785        VSETIVLI(REG_X0, 2, 0, 0, 0b011, 0b000)
4786#endif
4787
4788        /* v2 = (x[1] * y[1])[0] | (x[0] * y[0])[0]  */
4789        VCLMUL_VV(REG_V2, REG_V0, REG_V1)
4790        /* v3 = (x[1] * y[1])[1] | (x[0] * y[0])[1]  */
4791        VCLMULH_VV(REG_V3, REG_V0, REG_V1)
4792        /* V2 = R[2] | R[0], V3 = R[3] | R[1] */
4793
4794        /* SWAP 64-bit values from V1 into V6. V6 = V1[0] | V1[1] */
4795        VSLIDEDOWN_VI(REG_V6, REG_V1, 1)
4796        VSLIDEUP_VI(REG_V6, REG_V1, 1)
4797        /* (x[1] * y[0])[0] | (x[0] * y[1])[0]  */
4798        VCLMUL_VV(REG_V4, REG_V0, REG_V6)
4799        /* (x[1] * y[0])[1] | (x[0] * y[1])[1]  */
4800        VCLMULH_VV(REG_V5, REG_V0, REG_V6)
4801        /* V4 = R[1] | R[1], V5 = R[2] | R[2] */
4802
4803        VMV_V_V(REG_V1, REG_V3)
4804        VSLIDEDOWN_VI(REG_V0, REG_V2, 1)
4805        VSLIDEUP_VI(REG_V1, REG_V0, 1)
4806        /* V2 =  ---- | R[0], V3 = R[3] | ----, V1 = R[2] | R[1] */
4807
4808        VMV_V_V(REG_V6, REG_V4)
4809        /* V7 = ---- | ----, V6 = ---- | R[1] */
4810        VSLIDEDOWN_VI(REG_V7, REG_V4, 1)
4811        /* V7 = ---- | R[1], V6 = ---- | R[1] */
4812        VSLIDEUP_VI(REG_V6, REG_V5, 1)
4813        /* V7 = ---- | R[1], V6 = R[2] | R[1] */
4814        VSLIDEDOWN_VI(REG_V0, REG_V5, 1)
4815        VSLIDEUP_VI(REG_V7, REG_V0, 1)
4816        /* V7 = R[2] | R[1], V6 = R[2] | R[1] */
4817        VXOR_VV(REG_V1, REG_V1, REG_V6)
4818        VXOR_VV(REG_V1, REG_V1, REG_V7)
4819        /* V2 =  ---- | R[0], V3 = R[3] | ----, V1 = R[2] | R[1] */
4820        VSLIDEUP_VI(REG_V2, REG_V1, 1)
4821        VSLIDEDOWN_VI(REG_V5, REG_V3, 1)
4822        VSLIDEDOWN_VI(REG_V3, REG_V1, 1)
4823        VSLIDEUP_VI(REG_V3, REG_V5, 1)
4824        /* V2 =  R[1] | R[0], V3 = R[3] | R[2] */
4825
4826        /* Reduce */
4827        /* v0 = (R[3] * 0x87)[0] | (R[2] * 0x87)[0]  */
4828        VCLMUL_VV(REG_V0, REG_V3, REG_V8)
4829        /* v1 = (R[3] * 0x87)[1] | (R[2] * 0x87)[1]  */
4830        VCLMULH_VV(REG_V1, REG_V3, REG_V8)
4831        /* V0 = r[1] | r[0], V1 = r[2] | r[1] */
4832        VXOR_VV(REG_V4, REG_V4, REG_V4)
4833        VXOR_VV(REG_V2, REG_V2, REG_V0)
4834        VSLIDEUP_VI(REG_V4, REG_V1, 1)
4835        VXOR_VV(REG_V2, REG_V2, REG_V4)
4836        VSLIDEDOWN_VI(REG_V3, REG_V1, 1)
4837        /* v0 = ---- | (r[2] * 0x87)[0]  */
4838        VCLMUL_VV(REG_V0, REG_V3, REG_V8)
4839        /* v1 = ---- | (r[2] * 0x87)[1] */
4840        VCLMULH_VV(REG_V1, REG_V3, REG_V8)
4841        /* V0 = ---- | r[0] , V1 = ---- | r[1] */
4842        VSLIDEUP_VI(REG_V0, REG_V1, 1)
4843        /* V1 = R[1] | R[0] */
4844        VXOR_VV(REG_V2, REG_V2, REG_V0)
4845
4846        /* Reverse x. */
4847#ifdef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
4848        VBREV8(REG_V2, REG_V2)
4849#else
4850        VSETIVLI(REG_X0, 16, 0, 0, 0b000, 0b000)
4851
4852        /* Swap odd/even bits. */
4853        "li          t0, 0x55\n\t"
4854        VMV_V_X(REG_V4, REG_T0)
4855        VSRL_VI(REG_V0, REG_V2, 1)
4856        VAND_VV(REG_V2, REG_V2, REG_V4)
4857        VAND_VV(REG_V0, REG_V0, REG_V4)
4858        VSLL_VI(REG_V2, REG_V2, 1)
4859        VOR_VV(REG_V2, REG_V2, REG_V0)
4860        /* Swap pairs of bits. */
4861        "li          t0, 0x33\n\t"
4862        VMV_V_X(REG_V4, REG_T0)
4863        VSRL_VI(REG_V0, REG_V2, 2)
4864        VAND_VV(REG_V2, REG_V2, REG_V4)
4865        VAND_VV(REG_V0, REG_V0, REG_V4)
4866        VSLL_VI(REG_V2, REG_V2, 2)
4867        VOR_VV(REG_V2, REG_V2, REG_V0)
4868        /* Swap nibbles. */
4869        "li          t0, 0x0f\n\t"
4870        VMV_V_X(REG_V4, REG_T0)
4871        VSRL_VI(REG_V0, REG_V2, 4)
4872        VAND_VV(REG_V2, REG_V2, REG_V4)
4873        VAND_VV(REG_V0, REG_V0, REG_V4)
4874        VSLL_VI(REG_V2, REG_V2, 4)
4875        VOR_VV(REG_V2, REG_V2, REG_V0)
4876
4877        VSETIVLI(REG_X0, 2, 0, 0, 0b011, 0b000)
4878#endif
4879        VS1R_V(REG_V2, REG_T1)
4880        :
4881        : [x] "r" (x), [y] "r" (y), [red] "r" (red)
4882        : "memory", "t0", "t1", "t2"
4883    );
4884}
4885
4886/* GHASH Additional Authentication Data (AAD) and cipher text.
4887 *
4888 * @param [in]  gcm  GCM object.
4889 * @param [in]  a    Additional Authentication Data (AAD).
4890 * @param [in]  aSz  Size of AAD in bytes.
4891 * @param [in]  c    Cipher text.
4892 * @param [in]  cSz  Size of cipher text in bytes.
4893 * @param [out] s    Hash result.
4894 * @param [in]  sSz  Number of bytes to put into hash result.
4895 */
4896void GHASH(Gcm* gcm, const byte* a, word32 aSz, const byte* c, word32 cSz,
4897    byte* s, word32 sSz)
4898{
4899    byte x[WC_AES_BLOCK_SIZE];
4900    byte scratch[WC_AES_BLOCK_SIZE];
4901    word32 blocks, partial;
4902    byte* h;
4903
4904    if (gcm == NULL) {
4905        return;
4906    }
4907
4908    h = gcm->H;
4909    XMEMSET(x, 0, WC_AES_BLOCK_SIZE);
4910
4911    /* Hash in A, the Additional Authentication Data */
4912    if (aSz != 0 && a != NULL) {
4913        blocks = aSz / WC_AES_BLOCK_SIZE;
4914        partial = aSz % WC_AES_BLOCK_SIZE;
4915        while (blocks--) {
4916            xorbuf16(x, a);
4917            GMULT(x, h);
4918            a += WC_AES_BLOCK_SIZE;
4919        }
4920        if (partial != 0) {
4921            XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
4922            XMEMCPY(scratch, a, partial);
4923            xorbuf16(x, scratch);
4924            GMULT(x, h);
4925        }
4926    }
4927
4928    /* Hash in C, the Ciphertext */
4929    if (cSz != 0 && c != NULL) {
4930        blocks = cSz / WC_AES_BLOCK_SIZE;
4931        partial = cSz % WC_AES_BLOCK_SIZE;
4932        while (blocks--) {
4933            xorbuf16(x, c);
4934            GMULT(x, h);
4935            c += WC_AES_BLOCK_SIZE;
4936        }
4937        if (partial != 0) {
4938            XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
4939            XMEMCPY(scratch, c, partial);
4940            xorbuf16(x, scratch);
4941            GMULT(x, h);
4942        }
4943    }
4944
4945    /* Hash in the lengths of A and C in bits */
4946    FlattenSzInBits(&scratch[0], aSz);
4947    FlattenSzInBits(&scratch[8], cSz);
4948    xorbuf16(x, scratch);
4949    GMULT(x, h);
4950
4951    /* Copy the result into s. */
4952    XMEMCPY(s, x, sSz);
4953}
4954
4955#define HAVE_GHASH
4956
4957#elif defined(WOLFSSL_RISCV_CARRYLESS)
4958
4959/* Bottom half of carryless-multiplication: rd = (rs1 * rs2)[0..63]. */
4960#define CLMUL(rd, rs1, rs2)                                 \
4961    ASM_WORD(0b00001010000000000001000000110011 |           \
4962             (rd << 7) | (rs1 << 15) | (rs2 << 20))
4963/* Top half of carryless-multiplication: rd = (rs1 * rs2)[64..127]. */
4964#define CLMULH(rd, rs1, rs2)                                \
4965    ASM_WORD(0b00001010000000000011000000110011 |           \
4966             (rd << 7) | (rs1 << 15) | (rs2 << 20))
4967
4968/* GMULT, multiply in GF2, x and y into x.
4969 *
4970 * @param [in, out]  x  On in, value to GMULT.
4971 *                      On out, result of GMULT.
4972 * @param [in]       y  Value to GMULT.
4973 */
4974static void GMULT(byte* x, byte* y)
4975{
4976    __asm__ __volatile__ (
4977        "ld         t0, 0(%[x])\n\t"
4978        "ld         t1, 8(%[x])\n\t"
4979        "ld         t2, 0(%[y])\n\t"
4980        "ld         t3, 8(%[y])\n\t"
4981        /* Load reduction value into t6 */
4982        "li         t6, 0x87\n\t"
4983        /* Reverse x. y was reversed in wc_AesGcmSetKey. */
4984#ifdef WOLFSSL_RISCV_BIT_MANIPULATION
4985        BREV8(REG_T0, REG_T0)
4986        BREV8(REG_T1, REG_T1)
4987#else
4988        /* Swap odd-even bits. */
4989        "li          a4, 0x5555555555555555\n\t"
4990        "srli        a2, t0, 1\n\t"
4991        "srli        a3, t1, 1\n\t"
4992        "and         t0, t0, a4\n\t"
4993        "and         t1, t1, a4\n\t"
4994        "and         a2, a2, a4\n\t"
4995        "and         a3, a3, a4\n\t"
4996        "slli        t0, t0, 1\n\t"
4997        "slli        t1, t1, 1\n\t"
4998        "or          t0, t0, a2\n\t"
4999        "or          t1, t1, a3\n\t"
5000        /* Swap pairs. */
5001        "li          a4, 0x3333333333333333\n\t"
5002        "srli        a2, t0, 2\n\t"
5003        "srli        a3, t1, 2\n\t"
5004        "and         t0, t0, a4\n\t"
5005        "and         t1, t1, a4\n\t"
5006        "and         a2, a2, a4\n\t"
5007        "and         a3, a3, a4\n\t"
5008        "slli        t0, t0, 2\n\t"
5009        "slli        t1, t1, 2\n\t"
5010        "or          t0, t0, a2\n\t"
5011        "or          t1, t1, a3\n\t"
5012        /* Swap nibbles. */
5013        "li          a4, 0x0f0f0f0f0f0f0f0f\n\t"
5014        "srli        a2, t0, 4\n\t"
5015        "srli        a3, t1, 4\n\t"
5016        "and         t0, t0, a4\n\t"
5017        "and         t1, t1, a4\n\t"
5018        "and         a2, a2, a4\n\t"
5019        "and         a3, a3, a4\n\t"
5020        "slli        t0, t0, 4\n\t"
5021        "slli        t1, t1, 4\n\t"
5022        "or          t0, t0, a2\n\t"
5023        "or          t1, t1, a3\n\t"
5024#endif
5025
5026        /* r[0..1] = x[0] * y[0] */
5027        CLMUL(REG_A2, REG_T0, REG_T2)
5028        CLMULH(REG_A3, REG_T0, REG_T2)
5029        /* r[2..3] = x[1] * y[1] */
5030        CLMUL(REG_A4, REG_T1, REG_T3)
5031        CLMULH(REG_A5, REG_T1, REG_T3)
5032        /* r[1..2] ^= x[1] * y[0] */
5033        CLMUL(REG_T4, REG_T1, REG_T2)
5034        CLMULH(REG_T5, REG_T1, REG_T2)
5035        "xor        a3, a3, t4\n\t"
5036        "xor        a4, a4, t5\n\t"
5037        /* r[1..2] ^= x[0] * y[1] */
5038        CLMUL(REG_T4, REG_T0, REG_T3)
5039        CLMULH(REG_T5, REG_T0, REG_T3)
5040        "xor        a3, a3, t4\n\t"
5041        "xor        a4, a4, t5\n\t"
5042
5043        /* Reduce */
5044        CLMUL(REG_T4, REG_A5, REG_T6)
5045        CLMULH(REG_T5, REG_A5, REG_T6)
5046        "xor        a3, a3, t4\n\t"
5047        "xor        a4, a4, t5\n\t"
5048        CLMUL(REG_T4, REG_A4, REG_T6)
5049        CLMULH(REG_T5, REG_A4, REG_T6)
5050        "xor        t0, a2, t4\n\t"
5051        "xor        t1, a3, t5\n\t"
5052
5053        /* Reverse x. */
5054#ifdef WOLFSSL_RISCV_BIT_MANIPULATION
5055        BREV8(REG_T0, REG_T0)
5056        BREV8(REG_T1, REG_T1)
5057#else
5058        /* Swap odd-even bits. */
5059        "li          a4, 0x5555555555555555\n\t"
5060        "srli        a2, t0, 1\n\t"
5061        "srli        a3, t1, 1\n\t"
5062        "and         t0, t0, a4\n\t"
5063        "and         t1, t1, a4\n\t"
5064        "and         a2, a2, a4\n\t"
5065        "and         a3, a3, a4\n\t"
5066        "slli        t0, t0, 1\n\t"
5067        "slli        t1, t1, 1\n\t"
5068        "or          t0, t0, a2\n\t"
5069        "or          t1, t1, a3\n\t"
5070        /* Swap pairs. */
5071        "li          a4, 0x3333333333333333\n\t"
5072        "srli        a2, t0, 2\n\t"
5073        "srli        a3, t1, 2\n\t"
5074        "and         t0, t0, a4\n\t"
5075        "and         t1, t1, a4\n\t"
5076        "and         a2, a2, a4\n\t"
5077        "and         a3, a3, a4\n\t"
5078        "slli        t0, t0, 2\n\t"
5079        "slli        t1, t1, 2\n\t"
5080        "or          t0, t0, a2\n\t"
5081        "or          t1, t1, a3\n\t"
5082        /* Swap nibbles. */
5083        "li          a4, 0x0f0f0f0f0f0f0f0f\n\t"
5084        "srli        a2, t0, 4\n\t"
5085        "srli        a3, t1, 4\n\t"
5086        "and         t0, t0, a4\n\t"
5087        "and         t1, t1, a4\n\t"
5088        "and         a2, a2, a4\n\t"
5089        "and         a3, a3, a4\n\t"
5090        "slli        t0, t0, 4\n\t"
5091        "slli        t1, t1, 4\n\t"
5092        "or          t0, t0, a2\n\t"
5093        "or          t1, t1, a3\n\t"
5094#endif
5095        "sd         t0, 0(%[x])\n\t"
5096        "sd         t1, 8(%[x])\n\t"
5097        :
5098        : [x] "r" (x), [y] "r" (y)
5099        : "memory", "t0", "t1", "t2", "t3", "t4", "t5", "t6",
5100          "a2", "a3", "a4", "a5"
5101    );
5102}
5103
5104/* GHASH blocks of data.
5105 *
5106 * @param [in, out]  x       On in, value to GMULT.
5107 *                           On out, result of GMULT.
5108 * @param [in]       y       Value to GMULT.
5109 * @param [in]       in      Blocks of data to GHASH.
5110 * @param [in]       blocks  Number of blocks to GHASH.
5111 */
5112static void ghash_blocks(byte* x, byte* y, const byte* in, word32 blocks)
5113{
5114    __asm__ __volatile__ (
5115        "ld         t0, 0(%[x])\n\t"
5116        "ld         t1, 8(%[x])\n\t"
5117        "ld         t2, 0(%[y])\n\t"
5118        "ld         t3, 8(%[y])\n\t"
5119        /* Load reduction value into t6 */
5120        "li         t6, 0x87\n\t"
5121        /* Reverse x. y was reversed in wc_AesGcmSetKey. */
5122#ifdef WOLFSSL_RISCV_BIT_MANIPULATION
5123        BREV8(REG_T0, REG_T0)
5124        BREV8(REG_T1, REG_T1)
5125#else
5126        /* Swap odd-even bits. */
5127        "li          a4, 0x5555555555555555\n\t"
5128        "srli        a2, t0, 1\n\t"
5129        "srli        a3, t1, 1\n\t"
5130        "and         t0, t0, a4\n\t"
5131        "and         t1, t1, a4\n\t"
5132        "and         a2, a2, a4\n\t"
5133        "and         a3, a3, a4\n\t"
5134        "slli        t0, t0, 1\n\t"
5135        "slli        t1, t1, 1\n\t"
5136        "or          t0, t0, a2\n\t"
5137        "or          t1, t1, a3\n\t"
5138        /* Swap pairs. */
5139        "li          a4, 0x3333333333333333\n\t"
5140        "srli        a2, t0, 2\n\t"
5141        "srli        a3, t1, 2\n\t"
5142        "and         t0, t0, a4\n\t"
5143        "and         t1, t1, a4\n\t"
5144        "and         a2, a2, a4\n\t"
5145        "and         a3, a3, a4\n\t"
5146        "slli        t0, t0, 2\n\t"
5147        "slli        t1, t1, 2\n\t"
5148        "or          t0, t0, a2\n\t"
5149        "or          t1, t1, a3\n\t"
5150        /* Swap nibbles. */
5151        "li          a4, 0x0f0f0f0f0f0f0f0f\n\t"
5152        "srli        a2, t0, 4\n\t"
5153        "srli        a3, t1, 4\n\t"
5154        "and         t0, t0, a4\n\t"
5155        "and         t1, t1, a4\n\t"
5156        "and         a2, a2, a4\n\t"
5157        "and         a3, a3, a4\n\t"
5158        "slli        t0, t0, 4\n\t"
5159        "slli        t1, t1, 4\n\t"
5160        "or          t0, t0, a2\n\t"
5161        "or          t1, t1, a3\n\t"
5162#endif
5163
5164    "L_ghash_loop:\n\t"
5165        /* Load input block. */
5166        "ld          t5, 0(%[in])\n\t"
5167        "ld          a5, 8(%[in])\n\t"
5168        /* Reverse bits to match x. */
5169#ifdef WOLFSSL_RISCV_BIT_MANIPULATION
5170        BREV8(REG_T5, REG_T5)
5171        BREV8(REG_A5, REG_A5)
5172#else
5173        /* Swap odd-even bits. */
5174        "li          a4, 0x5555555555555555\n\t"
5175        "srli        a2, t5, 1\n\t"
5176        "srli        a3, a5, 1\n\t"
5177        "and         t5, t5, a4\n\t"
5178        "and         a5, a5, a4\n\t"
5179        "and         a2, a2, a4\n\t"
5180        "and         a3, a3, a4\n\t"
5181        "slli        t5, t5, 1\n\t"
5182        "slli        a5, a5, 1\n\t"
5183        "or          t5, t5, a2\n\t"
5184        "or          a5, a5, a3\n\t"
5185        /* Swap pairs. */
5186        "li          a4, 0x3333333333333333\n\t"
5187        "srli        a2, t5, 2\n\t"
5188        "srli        a3, a5, 2\n\t"
5189        "and         t5, t5, a4\n\t"
5190        "and         a5, a5, a4\n\t"
5191        "and         a2, a2, a4\n\t"
5192        "and         a3, a3, a4\n\t"
5193        "slli        t5, t5, 2\n\t"
5194        "slli        a5, a5, 2\n\t"
5195        "or          t5, t5, a2\n\t"
5196        "or          a5, a5, a3\n\t"
5197        /* Swap nibbles. */
5198        "li          a4, 0x0f0f0f0f0f0f0f0f\n\t"
5199        "srli        a2, t5, 4\n\t"
5200        "srli        a3, a5, 4\n\t"
5201        "and         t5, t5, a4\n\t"
5202        "and         a5, a5, a4\n\t"
5203        "and         a2, a2, a4\n\t"
5204        "and         a3, a3, a4\n\t"
5205        "slli        t5, t5, 4\n\t"
5206        "slli        a5, a5, 4\n\t"
5207        "or          t5, t5, a2\n\t"
5208        "or          a5, a5, a3\n\t"
5209#endif
5210        /* XOR input into x. */
5211        "xor         t0, t0, t5\n\t"
5212        "xor         t1, t1, a5\n\t"
5213
5214        /* r[0..1] = x[0] * y[0] */
5215        CLMUL(REG_A2, REG_T0, REG_T2)
5216        CLMULH(REG_A3, REG_T0, REG_T2)
5217        /* r[2..3] = x[1] * y[1] */
5218        CLMUL(REG_A4, REG_T1, REG_T3)
5219        CLMULH(REG_A5, REG_T1, REG_T3)
5220        /* r[1..2] ^= x[1] * y[0] */
5221        CLMUL(REG_T4, REG_T1, REG_T2)
5222        CLMULH(REG_T5, REG_T1, REG_T2)
5223        "xor        a3, a3, t4\n\t"
5224        "xor        a4, a4, t5\n\t"
5225        /* r[1..2] ^= x[0] * y[1] */
5226        CLMUL(REG_T4, REG_T0, REG_T3)
5227        CLMULH(REG_T5, REG_T0, REG_T3)
5228        "xor        a3, a3, t4\n\t"
5229        "xor        a4, a4, t5\n\t"
5230
5231        /* Reduce */
5232        CLMUL(REG_T4, REG_A5, REG_T6)
5233        CLMULH(REG_T5, REG_A5, REG_T6)
5234        "xor        a3, a3, t4\n\t"
5235        "xor        a4, a4, t5\n\t"
5236        CLMUL(REG_T4, REG_A4, REG_T6)
5237        CLMULH(REG_T5, REG_A4, REG_T6)
5238        "xor        t0, a2, t4\n\t"
5239        "xor        t1, a3, t5\n\t"
5240
5241        "addi        %[in], %[in], 16\n\t"
5242        "addi        %[blocks], %[blocks], -1\n\t"
5243        "bnez        %[blocks], L_ghash_loop\n\t"
5244
5245        /* Reverse x. */
5246#ifdef WOLFSSL_RISCV_BIT_MANIPULATION
5247        BREV8(REG_T0, REG_T0)
5248        BREV8(REG_T1, REG_T1)
5249#else
5250        /* Swap odd-even bits. */
5251        "li          a4, 0x5555555555555555\n\t"
5252        "srli        a2, t0, 1\n\t"
5253        "srli        a3, t1, 1\n\t"
5254        "and         t0, t0, a4\n\t"
5255        "and         t1, t1, a4\n\t"
5256        "and         a2, a2, a4\n\t"
5257        "and         a3, a3, a4\n\t"
5258        "slli        t0, t0, 1\n\t"
5259        "slli        t1, t1, 1\n\t"
5260        "or          t0, t0, a2\n\t"
5261        "or          t1, t1, a3\n\t"
5262        /* Swap pairs. */
5263        "li          a4, 0x3333333333333333\n\t"
5264        "srli        a2, t0, 2\n\t"
5265        "srli        a3, t1, 2\n\t"
5266        "and         t0, t0, a4\n\t"
5267        "and         t1, t1, a4\n\t"
5268        "and         a2, a2, a4\n\t"
5269        "and         a3, a3, a4\n\t"
5270        "slli        t0, t0, 2\n\t"
5271        "slli        t1, t1, 2\n\t"
5272        "or          t0, t0, a2\n\t"
5273        "or          t1, t1, a3\n\t"
5274        /* Swap nibbles. */
5275        "li          a4, 0x0f0f0f0f0f0f0f0f\n\t"
5276        "srli        a2, t0, 4\n\t"
5277        "srli        a3, t1, 4\n\t"
5278        "and         t0, t0, a4\n\t"
5279        "and         t1, t1, a4\n\t"
5280        "and         a2, a2, a4\n\t"
5281        "and         a3, a3, a4\n\t"
5282        "slli        t0, t0, 4\n\t"
5283        "slli        t1, t1, 4\n\t"
5284        "or          t0, t0, a2\n\t"
5285        "or          t1, t1, a3\n\t"
5286#endif
5287        "sd         t0, 0(%[x])\n\t"
5288        "sd         t1, 8(%[x])\n\t"
5289        : [in] "+r" (in), [blocks] "+r" (blocks)
5290        : [x] "r" (x), [y] "r" (y)
5291        : "memory", "t0", "t1", "t2", "t3", "t4", "t5", "t6",
5292          "a2", "a3", "a4", "a5"
5293    );
5294}
5295
5296/* GHASH Additional Authentication Data (AAD) and cipher text.
5297 *
5298 * @param [in]  gcm  GCM object.
5299 * @param [in]  a    Additional Authentication Data (AAD).
5300 * @param [in]  aSz  Size of AAD in bytes.
5301 * @param [in]  c    Cipher text.
5302 * @param [in]  cSz  Size of cipher text in bytes.
5303 * @param [out] s    Hash result.
5304 * @param [in]  sSz  Number of bytes to put into hash result.
5305 */
5306void GHASH(Gcm* gcm, const byte* a, word32 aSz, const byte* c, word32 cSz,
5307    byte* s, word32 sSz)
5308{
5309    if (gcm != NULL) {
5310        byte x[WC_AES_BLOCK_SIZE];
5311        byte scratch[WC_AES_BLOCK_SIZE];
5312        word32 blocks, partial;
5313        byte* h = gcm->H;
5314
5315        XMEMSET(x, 0, WC_AES_BLOCK_SIZE);
5316
5317        /* Hash in A, the Additional Authentication Data */
5318        if (aSz != 0 && a != NULL) {
5319            blocks = aSz / WC_AES_BLOCK_SIZE;
5320            partial = aSz % WC_AES_BLOCK_SIZE;
5321            if (blocks > 0) {
5322                ghash_blocks(x, h, a, blocks);
5323                a += blocks * WC_AES_BLOCK_SIZE;
5324            }
5325            if (partial != 0) {
5326                XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
5327                XMEMCPY(scratch, a, partial);
5328                xorbuf16(x, scratch);
5329                GMULT(x, h);
5330            }
5331        }
5332
5333        /* Hash in C, the Ciphertext */
5334        if (cSz != 0 && c != NULL) {
5335            blocks = cSz / WC_AES_BLOCK_SIZE;
5336            partial = cSz % WC_AES_BLOCK_SIZE;
5337            if (blocks > 0) {
5338                ghash_blocks(x, h, c, blocks);
5339                c += blocks * WC_AES_BLOCK_SIZE;
5340            }
5341            if (partial != 0) {
5342                XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
5343                XMEMCPY(scratch, c, partial);
5344                xorbuf16(x, scratch);
5345                GMULT(x, h);
5346            }
5347        }
5348
5349        /* Hash in the lengths of A and C in bits */
5350        FlattenSzInBits(&scratch[0], aSz);
5351        FlattenSzInBits(&scratch[8], cSz);
5352        xorbuf16(x, scratch);
5353        GMULT(x, h);
5354
5355        /* Copy the result into s. */
5356        XMEMCPY(s, x, sSz);
5357    }
5358}
5359
5360#define HAVE_GHASH
5361
5362#endif /* !WOLFSSL_RISCV_VECTOR_GCM */
5363
5364#ifdef WOLFSSL_RISCV_VECTOR_CRYPTO_ASM
5365#ifdef WOLFSSL_RISCV_VECTOR_GCM
5366/* START script replace AES-GCM RISC-V 64 with hardware vector crypto */
5367#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
5368static const word32 rev_idx[4] = {
5369    0x00010203, 0x04050607, 0x08090a0b, 0x0c0d0e0f
5370};
5371#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
5372
5373#ifdef WOLFSSL_AES_128
5374/* Encrypt data using AES-128-GCM.
5375 *
5376 * @param [in]  aes      AES object.
5377 * @param [out] out      Encrypted data.
5378 * @param [in]  in       Data to encrypt and GHASH.
5379 * @param [in]  sz       Number of bytes of data.
5380 * @param [in]  nonce    Nonce used to calculate first IV.
5381 * @param [in]  nonceSz  Length of nonce in bytes.
5382 * @param [out] tag      Authentication tag.
5383 * @param [in]  tagSz    Length of authentication tag in bytes.
5384 * @param [in]  aad      Additional Authentication Data (AAD).
5385 * @param [in]  aadSz    Length of AAD in bytes.
5386 */
5387static void Aes128GcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz,
5388    const byte* nonce, word32 nonceSz, byte* tag, word32 tagSz,
5389    const byte* aad, word32 aadSz)
5390{
5391    byte counter[WC_AES_BLOCK_SIZE];
5392    byte scratch[WC_AES_BLOCK_SIZE];
5393    /* Noticed different optimization levels treated head of array different.
5394     * Some cases was stack pointer plus offset others was a register containing
5395     * address. To make uniform for passing in to inline assembly code am using
5396     * pointers to the head of each local array.
5397     */
5398    byte* ctr  = counter;
5399    byte* key = (byte*)aes->key;
5400
5401    XMEMSET(counter, 0, WC_AES_BLOCK_SIZE);
5402    if (nonceSz == GCM_NONCE_MID_SZ) {
5403        XMEMCPY(counter, nonce, GCM_NONCE_MID_SZ);
5404        counter[WC_AES_BLOCK_SIZE - 1] = 1;
5405    }
5406    else {
5407#ifdef OPENSSL_EXTRA
5408        word32 aadTemp = aes->gcm.aadLen;
5409        aes->gcm.aadLen = 0;
5410#endif
5411        GHASH(&aes->gcm, NULL, 0, nonce, nonceSz, counter, WC_AES_BLOCK_SIZE);
5412#ifdef OPENSSL_EXTRA
5413        aes->gcm.aadLen = aadTemp;
5414#endif
5415    }
5416
5417    __asm__ __volatile__ (
5418        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
5419
5420        /* X=0, get H */
5421        VXOR_VV(REG_V18, REG_V18, REG_V18)
5422        "mv         t0, %[h]\n\t"
5423        VL1RE32_V(REG_V19, REG_T0)
5424
5425        /* Hash in AAD, the Additional Authentication Data */
5426        "beqz       %[aSz], L_aes_gcm_128_encrypt_ghash_aad_done\n\t"
5427        "beqz       %[aad], L_aes_gcm_128_encrypt_ghash_aad_done\n\t"
5428
5429        "srli       t1, %[aSz], 4\n\t"
5430        "beqz       t1, L_aes_gcm_128_encrypt_ghash_aad_blocks_done\n\t"
5431
5432      "L_aes_gcm_128_encrypt_ghash_aad_loop:\n\t"
5433        "mv         t0, %[aad]\n\t"
5434        VL1RE32_V(REG_V17, REG_T0)
5435        VGHSH_VV(REG_V18, REG_V17, REG_V19)
5436        "addi       %[aad], %[aad], 16\n\t"
5437        "addi       t1, t1, -1\n\t"
5438        "bnez       t1, L_aes_gcm_128_encrypt_ghash_aad_loop\n\t"
5439      "L_aes_gcm_128_encrypt_ghash_aad_blocks_done:\n\t"
5440        "andi       t1, %[aSz], 0xf\n\t"
5441        "beqz       t1, L_aes_gcm_128_encrypt_ghash_aad_done\n\t"
5442        VXOR_VV(REG_V17, REG_V17, REG_V17)
5443        "mv         t0, %[scratch]\n\t"
5444        VS1R_V(REG_V17, REG_T0)
5445        "mv         t2, t1\n\t"
5446      "L_aes_gcm_128_encrypt_ghash_aad_load_byte:\n\t"
5447        "lb         t0, (%[aad])\n\t"
5448        "sb         t0, (%[scratch])\n\t"
5449        "addi       %[aad], %[aad], 1\n\t"
5450        "addi       %[scratch], %[scratch], 1\n\t"
5451        "addi       t2, t2, -1\n\t"
5452        "bnez       t2, L_aes_gcm_128_encrypt_ghash_aad_load_byte\n\t"
5453        "sub        %[scratch], %[scratch], t1\n\t"
5454        "mv         t0, %[scratch]\n\t"
5455        VL1RE32_V(REG_V17, REG_T0)
5456        VGHSH_VV(REG_V18, REG_V17, REG_V19)
5457      "L_aes_gcm_128_encrypt_ghash_aad_done:\n\t"
5458        /* Done Hash in AAD */
5459
5460#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
5461        "mv         t0, %[rev_idx]\n\t"
5462        VL1RE32_V(REG_V15, REG_T0)
5463#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
5464        /* Load the counter. */
5465        "mv         t0, %[ctr]\n\t"
5466        VL1RE32_V(REG_V16, REG_T0)
5467#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
5468        VSLIDEDOWN_VI(REG_V20, REG_V16, 3)
5469#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
5470        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
5471        VRGATHER_VV(REG_V21, REG_V15, REG_V20)
5472        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
5473        VMV_V_V(REG_V20, REG_V21)
5474#else
5475        VREV8(REG_V20, REG_V20)
5476#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
5477#else
5478        "lw         t3, 12(%[ctr])\n\t"
5479        "slli       t3, t3, 32\n\t"
5480        REV8(REG_T3, REG_T3)
5481#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
5482
5483        /* Load key[0..7]. */
5484        "mv         t0, %[key]\n\t"
5485        VL8RE32_V(REG_V0, REG_T0)
5486        /* Load key[8..9]. */
5487        "addi       t0, t0, 128\n\t"
5488        VL2RE32_V(REG_V8, REG_T0)
5489        /* Load last round's key */
5490        "addi       t0, %[key], 224\n\t"
5491        VL1RE32_V(REG_V10, REG_T0)
5492
5493        "beqz       %[sz], L_aes_gcm_128_encrypt_blocks_done\n\t"
5494        "srli       t4, %[sz], 6\n\t"
5495        "beqz       t4, L_aes_gcm_128_encrypt_x4_blocks_done\n\t"
5496
5497        /* Calculate H^[1-4] - GMULT partials */
5498        VMV_V_V(REG_V21, REG_V19)
5499        VMV_V_V(REG_V22, REG_V19)
5500        /* Multiply H * H => H^2 */
5501        VGMUL_VV(REG_V21, REG_V19)
5502        VMV_V_V(REG_V23, REG_V21)
5503        /* Multiply H * H => H^3 */
5504        VGMUL_VV(REG_V22, REG_V21)
5505        /* Multiply H^2 * H^2 => H^4 */
5506        VGMUL_VV(REG_V23, REG_V21)
5507
5508      "L_aes_gcm_128_encrypt_x4_block_loop:\n\t"
5509        /* Calculate next 4 counters (+1-4) */
5510#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
5511        VMV_V_V(REG_V24, REG_V16)
5512        VMV_V_V(REG_V25, REG_V16)
5513        VMV_V_V(REG_V26, REG_V16)
5514        VMV_V_V(REG_V27, REG_V16)
5515        VADD_VI(REG_V28, REG_V20, 1)
5516        VADD_VI(REG_V29, REG_V20, 2)
5517        VADD_VI(REG_V30, REG_V20, 3)
5518        VADD_VI(REG_V20, REG_V20, 4)
5519#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
5520        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
5521        VRGATHER_VV(REG_V17, REG_V15, REG_V28)
5522        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
5523        VMV_V_V(REG_V28, REG_V17)
5524#else
5525        VREV8(REG_V28, REG_V28)
5526#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
5527#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
5528        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
5529        VRGATHER_VV(REG_V17, REG_V15, REG_V29)
5530        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
5531        VMV_V_V(REG_V29, REG_V17)
5532#else
5533        VREV8(REG_V29, REG_V29)
5534#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
5535#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
5536        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
5537        VRGATHER_VV(REG_V17, REG_V15, REG_V30)
5538        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
5539        VMV_V_V(REG_V30, REG_V17)
5540#else
5541        VREV8(REG_V30, REG_V30)
5542#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
5543#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
5544        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
5545        VRGATHER_VV(REG_V31, REG_V15, REG_V20)
5546        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
5547#else
5548        VREV8(REG_V31, REG_V20)
5549#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
5550#else
5551        "addi       t0, t3, 1\n\t"
5552        VMV_V_V(REG_V24, REG_V16)
5553        "addi       t1, t3, 2\n\t"
5554        VMV_V_V(REG_V25, REG_V16)
5555        "addi       t2, t3, 3\n\t"
5556        VMV_V_V(REG_V26, REG_V16)
5557        "slli       t0, t0, 32\n\t"
5558        VMV_V_V(REG_V27, REG_V16)
5559        "slli       t1, t1, 32\n\t"
5560        "slli       t2, t2, 32\n\t"
5561        REV8(REG_T0, REG_T0)
5562        REV8(REG_T1, REG_T1)
5563        REV8(REG_T2, REG_T2)
5564        "addi       t3, t3, 4\n\t"
5565        VMV_V_X(REG_V28, REG_T0)
5566        "slli       t0, t3, 32\n\t"
5567        VMV_V_X(REG_V29, REG_T1)
5568        REV8(REG_T0, REG_T0)
5569        VMV_V_X(REG_V30, REG_T2)
5570        VMV_V_X(REG_V31, REG_T0)
5571#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
5572        VSLIDEUP_VI(REG_V24, REG_V28, 3)
5573        VSLIDEUP_VI(REG_V25, REG_V29, 3)
5574        VSLIDEUP_VI(REG_V26, REG_V30, 3)
5575        VSLIDEUP_VI(REG_V27, REG_V31, 3)
5576
5577        VAESZ_VS(REG_V24, REG_V0)
5578        VAESZ_VS(REG_V25, REG_V0)
5579        VAESZ_VS(REG_V26, REG_V0)
5580        VAESZ_VS(REG_V27, REG_V0)
5581        VAESEM_VS(REG_V24, REG_V1)
5582        VAESEM_VS(REG_V24, REG_V2)
5583        VAESEM_VS(REG_V24, REG_V3)
5584        VAESEM_VS(REG_V24, REG_V4)
5585        VAESEM_VS(REG_V24, REG_V5)
5586        VAESEM_VS(REG_V24, REG_V6)
5587        VAESEM_VS(REG_V24, REG_V7)
5588        VAESEM_VS(REG_V24, REG_V8)
5589        VAESEM_VS(REG_V24, REG_V9)
5590        VAESEM_VS(REG_V25, REG_V1)
5591        VAESEM_VS(REG_V25, REG_V2)
5592        VAESEM_VS(REG_V25, REG_V3)
5593        VAESEM_VS(REG_V25, REG_V4)
5594        VAESEM_VS(REG_V25, REG_V5)
5595        VAESEM_VS(REG_V25, REG_V6)
5596        VAESEM_VS(REG_V25, REG_V7)
5597        VAESEM_VS(REG_V25, REG_V8)
5598        VAESEM_VS(REG_V25, REG_V9)
5599        VAESEM_VS(REG_V26, REG_V1)
5600        VAESEM_VS(REG_V26, REG_V2)
5601        VAESEM_VS(REG_V26, REG_V3)
5602        VAESEM_VS(REG_V26, REG_V4)
5603        VAESEM_VS(REG_V26, REG_V5)
5604        VAESEM_VS(REG_V26, REG_V6)
5605        VAESEM_VS(REG_V26, REG_V7)
5606        VAESEM_VS(REG_V26, REG_V8)
5607        VAESEM_VS(REG_V26, REG_V9)
5608        VAESEM_VS(REG_V27, REG_V1)
5609        VAESEM_VS(REG_V27, REG_V2)
5610        VAESEM_VS(REG_V27, REG_V3)
5611        VAESEM_VS(REG_V27, REG_V4)
5612        VAESEM_VS(REG_V27, REG_V5)
5613        VAESEM_VS(REG_V27, REG_V6)
5614        VAESEM_VS(REG_V27, REG_V7)
5615        VAESEM_VS(REG_V27, REG_V8)
5616        VAESEM_VS(REG_V27, REG_V9)
5617        VAESEF_VS(REG_V24, REG_V10)
5618        VAESEF_VS(REG_V25, REG_V10)
5619        VAESEF_VS(REG_V26, REG_V10)
5620        VAESEF_VS(REG_V27, REG_V10)
5621
5622        /* Load input. */
5623        "mv        t0, %[in]\n\t"
5624        VL4RE32_V(REG_V28, REG_T0)
5625        VXOR_VV(REG_V28, REG_V24, REG_V28)
5626        VXOR_VV(REG_V29, REG_V25, REG_V29)
5627        VXOR_VV(REG_V30, REG_V26, REG_V30)
5628        VXOR_VV(REG_V31, REG_V27, REG_V31)
5629        /* Store output. */
5630        "mv         t0, %[out]\n\t"
5631        VS4R_V(REG_V28, REG_T0)
5632        VGMUL_VV(REG_V28, REG_V23)
5633        VGMUL_VV(REG_V29, REG_V22)
5634        VGMUL_VV(REG_V30, REG_V21)
5635        VGMUL_VV(REG_V31, REG_V19)
5636        VXOR_VV(REG_V18, REG_V18, REG_V28)
5637        VXOR_VV(REG_V18, REG_V18, REG_V29)
5638        VXOR_VV(REG_V18, REG_V18, REG_V30)
5639        VXOR_VV(REG_V18, REG_V18, REG_V31)
5640        "addi        %[in], %[in], 64\n\t"
5641        "addi        %[out], %[out], 64\n\t"
5642        /* Loop if more elements to process. */
5643        "addi       t4, t4, -1\n\t"
5644        "bnez       t4, L_aes_gcm_128_encrypt_x4_block_loop\n\t"
5645        "andi       %[sz], %[sz], 0x3f\n\t"
5646
5647      "L_aes_gcm_128_encrypt_x4_blocks_done:\n\t"
5648        "srli       t2, %[sz], 4\n\t"
5649        "beqz       t2, L_aes_gcm_128_encrypt_blocks_done\n\t"
5650
5651      "L_aes_gcm_128_encrypt_block_loop:\n\t"
5652#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
5653        VADD_VI(REG_V20, REG_V20, 1)
5654#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
5655        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
5656        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
5657        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
5658#else
5659        VREV8(REG_V17, REG_V20)
5660#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
5661        VMV_V_V(REG_V27, REG_V16)
5662        VSLIDEUP_VI(REG_V27, REG_V17, 3)
5663#else
5664        "addi       t3, t3, 1\n\t"
5665        "slli       t0, t3, 32\n\t"
5666        REV8(REG_T0, REG_T0)
5667        VMV_V_X(REG_V17, REG_T0)
5668        VMV_V_V(REG_V27, REG_V16)
5669        VSLIDEUP_VI(REG_V27, REG_V17, 3)
5670#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
5671
5672        VAESZ_VS(REG_V27, REG_V0)
5673        VAESEM_VS(REG_V27, REG_V1)
5674        VAESEM_VS(REG_V27, REG_V2)
5675        VAESEM_VS(REG_V27, REG_V3)
5676        VAESEM_VS(REG_V27, REG_V4)
5677        VAESEM_VS(REG_V27, REG_V5)
5678        VAESEM_VS(REG_V27, REG_V6)
5679        VAESEM_VS(REG_V27, REG_V7)
5680        VAESEM_VS(REG_V27, REG_V8)
5681        VAESEM_VS(REG_V27, REG_V9)
5682        VAESEF_VS(REG_V27, REG_V10)
5683
5684        /* Load input. */
5685        "mv         t0, %[in]\n\t"
5686        VL1RE32_V(REG_V17, REG_T0)
5687        VXOR_VV(REG_V27, REG_V27, REG_V17)
5688        VGHSH_VV(REG_V18, REG_V27, REG_V19)
5689        /* Store output. */
5690        "mv         t0, %[out]\n\t"
5691        VS1R_V(REG_V27, REG_T0)
5692
5693        "addi        %[in], %[in], 16\n\t"
5694        "addi        %[out], %[out], 16\n\t"
5695        /* Loop if more elements to process. */
5696        "addi       t2, t2, -1\n\t"
5697        "bnez       t2, L_aes_gcm_128_encrypt_block_loop\n\t"
5698
5699      "L_aes_gcm_128_encrypt_blocks_done:\n\t"
5700        "andi       t2, %[sz], 0xf\n\t"
5701        "beqz       t2, L_aes_gcm_128_encrypt_done\n\t"
5702
5703        VXOR_VV(REG_V17, REG_V17, REG_V17)
5704        "mv         t0, %[scratch]\n\t"
5705        VS1R_V(REG_V17, REG_T0)
5706        "mv         t1, t2\n\t"
5707      "L_aes_gcm_128_encrypt_load_byte:\n\t"
5708        "lb         t0, (%[in])\n\t"
5709        "sb         t0, (%[scratch])\n\t"
5710        "addi       %[in], %[in], 1\n\t"
5711        "addi       %[scratch], %[scratch], 1\n\t"
5712        "addi       t1, t1, -1\n\t"
5713        "bnez       t1, L_aes_gcm_128_encrypt_load_byte\n\t"
5714        "sub        %[scratch], %[scratch], t2\n\t"
5715
5716        /* Encrypt counter for partial block. */
5717#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
5718        VADD_VI(REG_V20, REG_V20, 1)
5719#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
5720        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
5721        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
5722        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
5723#else
5724        VREV8(REG_V17, REG_V20)
5725#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
5726        VMV_V_V(REG_V27, REG_V16)
5727        VSLIDEUP_VI(REG_V27, REG_V17, 3)
5728#else
5729        "addi       t3, t3, 1\n\t"
5730        "slli       t0, t3, 32\n\t"
5731        REV8(REG_T0, REG_T0)
5732        VMV_V_X(REG_V17, REG_T0)
5733        VMV_V_V(REG_V27, REG_V16)
5734        VSLIDEUP_VI(REG_V27, REG_V17, 3)
5735#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
5736
5737        VAESZ_VS(REG_V27, REG_V0)
5738        VAESEM_VS(REG_V27, REG_V1)
5739        VAESEM_VS(REG_V27, REG_V2)
5740        VAESEM_VS(REG_V27, REG_V3)
5741        VAESEM_VS(REG_V27, REG_V4)
5742        VAESEM_VS(REG_V27, REG_V5)
5743        VAESEM_VS(REG_V27, REG_V6)
5744        VAESEM_VS(REG_V27, REG_V7)
5745        VAESEM_VS(REG_V27, REG_V8)
5746        VAESEM_VS(REG_V27, REG_V9)
5747        VAESEF_VS(REG_V27, REG_V10)
5748
5749        /* Load scratch. */
5750        "mv         t0, %[scratch]\n\t"
5751        VL1RE32_V(REG_V17, REG_T0)
5752        VXOR_VV(REG_V27, REG_V27, REG_V17)
5753        /* Store scratch. */
5754        VS1R_V(REG_V27, REG_T0)
5755        "mv         t1, t2\n\t"
5756      "L_aes_gcm_128_encrypt_store_byte:\n\t"
5757        "lb         t0, (%[scratch])\n\t"
5758        "sb         t0, (%[out])\n\t"
5759        "addi       %[scratch], %[scratch], 1\n\t"
5760        "addi       %[out], %[out], 1\n\t"
5761        "addi       t1, t1, -1\n\t"
5762        "bnez       t1, L_aes_gcm_128_encrypt_store_byte\n\t"
5763        "li         t1, 16\n\t"
5764        "sub        t1, t1, t2\n\t"
5765      "L_aes_gcm_128_encrypt_zero_byte:\n\t"
5766        "sb         x0, (%[scratch])\n\t"
5767        "addi       %[scratch], %[scratch], 1\n\t"
5768        "addi       t1, t1, -1\n\t"
5769        "bnez       t1, L_aes_gcm_128_encrypt_zero_byte\n\t"
5770        "addi       %[scratch], %[scratch], -16\n\t"
5771        "mv         t0, %[scratch]\n\t"
5772        VL1RE32_V(REG_V17, REG_T0)
5773        VGHSH_VV(REG_V18, REG_V17, REG_V19)
5774
5775      "L_aes_gcm_128_encrypt_done:\n\t"
5776
5777        /* Hash in the lengths of A and C in bits */
5778#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
5779        /* aSz is only 32-bits */
5780        /* Multiply by 8 do get size in bits. */
5781        "slli       %[aSz], %[aSz], 3\n\t"
5782        "srli       t0, %[aSz], 32\n\t"
5783        "srli       t1, %[aSz], 24\n\t"
5784        "srli       t2, %[aSz], 16\n\t"
5785        "srli       t3, %[aSz], 8\n\t"
5786        /* Top 3 bytes are 0. */
5787        "sh         x0    , 0(%[scratch])\n\t"
5788        "sb         x0    , 2(%[scratch])\n\t"
5789        "sb         t0    , 3(%[scratch])\n\t"
5790        "sb         t1    , 4(%[scratch])\n\t"
5791        "sb         t2    , 5(%[scratch])\n\t"
5792        "sb         t3    , 6(%[scratch])\n\t"
5793        "sb         %[aSz], 7(%[scratch])\n\t"
5794        /* sz is only 32-bits */
5795        /* Multiply by 8 do get size in bits. */
5796        "slli       %[sz], %[sz], 3\n\t"
5797        "srli       t0, %[sz], 32\n\t"
5798        "srli       t1, %[sz], 24\n\t"
5799        "srli       t2, %[sz], 16\n\t"
5800        "srli       t3, %[sz], 8\n\t"
5801        /* Top 3 bytes are 0. */
5802        "sh         x0   ,  8(%[scratch])\n\t"
5803        "sb         x0   , 10(%[scratch])\n\t"
5804        "sb         t0   , 11(%[scratch])\n\t"
5805        "sb         t1   , 12(%[scratch])\n\t"
5806        "sb         t2   , 13(%[scratch])\n\t"
5807        "sb         t3   , 14(%[scratch])\n\t"
5808        "sb         %[sz], 15(%[scratch])\n\t"
5809#else
5810        "slli       t0, %[aSz], 3\n\t"
5811        REV8(REG_T0, REG_T0)
5812        "sd         t0, 0(%[scratch])\n\t"
5813        "slli       t0, %[sz], 3\n\t"
5814        REV8(REG_T0, REG_T0)
5815        "sd         t0, 8(%[scratch])\n\t"
5816#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
5817        "mv         t0, %[scratch]\n\t"
5818        VL1RE32_V(REG_V17, REG_T0)
5819        VGHSH_VV(REG_V18, REG_V17, REG_V19)
5820
5821        VAESZ_VS(REG_V16, REG_V0)
5822        VAESEM_VS(REG_V16, REG_V1)
5823        VAESEM_VS(REG_V16, REG_V2)
5824        VAESEM_VS(REG_V16, REG_V3)
5825        VAESEM_VS(REG_V16, REG_V4)
5826        VAESEM_VS(REG_V16, REG_V5)
5827        VAESEM_VS(REG_V16, REG_V6)
5828        VAESEM_VS(REG_V16, REG_V7)
5829        VAESEM_VS(REG_V16, REG_V8)
5830        VAESEM_VS(REG_V16, REG_V9)
5831        VAESEF_VS(REG_V16, REG_V10)
5832        VXOR_VV(REG_V18, REG_V18, REG_V16)
5833
5834        "li         t1, 16\n\t"
5835        "blt        %[tagSz], t1, L_aes_gcm_128_encrypt_tag_small\n\t"
5836        "mv         t0, %[tag]\n\t"
5837        VS1R_V(REG_V18, REG_T0)
5838        "beqz       x0, L_aes_gcm_128_encrypt_tag_done\n\t"
5839      "L_aes_gcm_128_encrypt_tag_small:\n\t"
5840        "mv         t0, %[scratch]\n\t"
5841        VS1R_V(REG_V18, REG_T0)
5842        "mv         t1, %[tagSz]\n\t"
5843      "L_aes_gcm_128_encrypt_store_tag_byte:\n\t"
5844        "lb         t0, (%[scratch])\n\t"
5845        "sb         t0, (%[tag])\n\t"
5846        "addi       %[scratch], %[scratch], 1\n\t"
5847        "addi       %[tag], %[tag], 1\n\t"
5848        "addi       t1, t1, -1\n\t"
5849        "bnez       t1, L_aes_gcm_128_encrypt_store_tag_byte\n\t"
5850      "L_aes_gcm_128_encrypt_tag_done:\n\t"
5851
5852        : [out] "+r" (out), [in] "+r" (in), [key] "+r" (key),
5853          [aSz] "+r" (aadSz), [aad] "+r" (aad), [sz] "+r" (sz)
5854        : [ctr] "r" (ctr), [scratch] "r" (scratch),
5855          [h] "r" (aes->gcm.H), [tag] "r" (tag), [tagSz] "r" (tagSz)
5856#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
5857          , [rev_idx] "r" (rev_idx)
5858#endif
5859        : "memory", "t0", "t1", "t2", "t3", "t4"
5860    );
5861
5862#ifdef OPENSSL_EXTRA
5863    if ((tag != NULL) && (in != NULL) && (sz != 0)) {
5864        /* store AAD size for next call */
5865        aes->gcm.aadLen = aadSz;
5866    }
5867#endif
5868}
5869#endif /*  WOLFSSL_AES_128 */
5870
5871#ifdef WOLFSSL_AES_192
5872/* Encrypt data using AES-192-GCM.
5873 *
5874 * @param [in]  aes      AES object.
5875 * @param [out] out      Encrypted data.
5876 * @param [in]  in       Data to encrypt and GHASH.
5877 * @param [in]  sz       Number of bytes of data.
5878 * @param [in]  nonce    Nonce used to calculate first IV.
5879 * @param [in]  nonceSz  Length of nonce in bytes.
5880 * @param [out] tag      Authentication tag.
5881 * @param [in]  tagSz    Length of authentication tag in bytes.
5882 * @param [in]  aad      Additional Authentication Data (AAD).
5883 * @param [in]  aadSz    Length of AAD in bytes.
5884 */
5885static void Aes192GcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz,
5886    const byte* nonce, word32 nonceSz, byte* tag, word32 tagSz,
5887    const byte* aad, word32 aadSz)
5888{
5889    byte counter[WC_AES_BLOCK_SIZE];
5890    byte scratch[WC_AES_BLOCK_SIZE];
5891    /* Noticed different optimization levels treated head of array different.
5892     * Some cases was stack pointer plus offset others was a register containing
5893     * address. To make uniform for passing in to inline assembly code am using
5894     * pointers to the head of each local array.
5895     */
5896    byte* ctr  = counter;
5897    byte* key = (byte*)aes->key;
5898
5899    XMEMSET(counter, 0, WC_AES_BLOCK_SIZE);
5900    if (nonceSz == GCM_NONCE_MID_SZ) {
5901        XMEMCPY(counter, nonce, GCM_NONCE_MID_SZ);
5902        counter[WC_AES_BLOCK_SIZE - 1] = 1;
5903    }
5904    else {
5905#ifdef OPENSSL_EXTRA
5906        word32 aadTemp = aes->gcm.aadLen;
5907        aes->gcm.aadLen = 0;
5908#endif
5909        GHASH(&aes->gcm, NULL, 0, nonce, nonceSz, counter, WC_AES_BLOCK_SIZE);
5910#ifdef OPENSSL_EXTRA
5911        aes->gcm.aadLen = aadTemp;
5912#endif
5913    }
5914
5915    __asm__ __volatile__ (
5916        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
5917
5918        /* X=0, get H */
5919        VXOR_VV(REG_V18, REG_V18, REG_V18)
5920        "mv         t0, %[h]\n\t"
5921        VL1RE32_V(REG_V19, REG_T0)
5922
5923        /* Hash in AAD, the Additional Authentication Data */
5924        "beqz       %[aSz], L_aes_gcm_192_encrypt_ghash_aad_done\n\t"
5925        "beqz       %[aad], L_aes_gcm_192_encrypt_ghash_aad_done\n\t"
5926
5927        "srli       t1, %[aSz], 4\n\t"
5928        "beqz       t1, L_aes_gcm_192_encrypt_ghash_aad_blocks_done\n\t"
5929
5930      "L_aes_gcm_192_encrypt_ghash_aad_loop:\n\t"
5931        "mv         t0, %[aad]\n\t"
5932        VL1RE32_V(REG_V17, REG_T0)
5933        VGHSH_VV(REG_V18, REG_V17, REG_V19)
5934        "addi       %[aad], %[aad], 16\n\t"
5935        "addi       t1, t1, -1\n\t"
5936        "bnez       t1, L_aes_gcm_192_encrypt_ghash_aad_loop\n\t"
5937      "L_aes_gcm_192_encrypt_ghash_aad_blocks_done:\n\t"
5938        "andi       t1, %[aSz], 0xf\n\t"
5939        "beqz       t1, L_aes_gcm_192_encrypt_ghash_aad_done\n\t"
5940        VXOR_VV(REG_V17, REG_V17, REG_V17)
5941        "mv         t0, %[scratch]\n\t"
5942        VS1R_V(REG_V17, REG_T0)
5943        "mv         t2, t1\n\t"
5944      "L_aes_gcm_192_encrypt_ghash_aad_load_byte:\n\t"
5945        "lb         t0, (%[aad])\n\t"
5946        "sb         t0, (%[scratch])\n\t"
5947        "addi       %[aad], %[aad], 1\n\t"
5948        "addi       %[scratch], %[scratch], 1\n\t"
5949        "addi       t2, t2, -1\n\t"
5950        "bnez       t2, L_aes_gcm_192_encrypt_ghash_aad_load_byte\n\t"
5951        "sub        %[scratch], %[scratch], t1\n\t"
5952        "mv         t0, %[scratch]\n\t"
5953        VL1RE32_V(REG_V17, REG_T0)
5954        VGHSH_VV(REG_V18, REG_V17, REG_V19)
5955      "L_aes_gcm_192_encrypt_ghash_aad_done:\n\t"
5956        /* Done Hash in AAD */
5957
5958#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
5959        "mv         t0, %[rev_idx]\n\t"
5960        VL1RE32_V(REG_V15, REG_T0)
5961#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
5962        /* Load the counter. */
5963        "mv         t0, %[ctr]\n\t"
5964        VL1RE32_V(REG_V16, REG_T0)
5965#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
5966        VSLIDEDOWN_VI(REG_V20, REG_V16, 3)
5967#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
5968        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
5969        VRGATHER_VV(REG_V21, REG_V15, REG_V20)
5970        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
5971        VMV_V_V(REG_V20, REG_V21)
5972#else
5973        VREV8(REG_V20, REG_V20)
5974#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
5975#else
5976        "lw         t3, 12(%[ctr])\n\t"
5977        "slli       t3, t3, 32\n\t"
5978        REV8(REG_T3, REG_T3)
5979#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
5980
5981        /* Load key[0..7]. */
5982        "mv         t0, %[key]\n\t"
5983        VL8RE32_V(REG_V0, REG_T0)
5984        /* Load key[8..11]. */
5985        "addi       t0, t0, 128\n\t"
5986        VL4RE32_V(REG_V8, REG_T0)
5987        /* Load last round's key */
5988        "addi       t0, %[key], 224\n\t"
5989        VL1RE32_V(REG_V12, REG_T0)
5990
5991        "beqz       %[sz], L_aes_gcm_192_encrypt_blocks_done\n\t"
5992        "srli       t4, %[sz], 6\n\t"
5993        "beqz       t4, L_aes_gcm_192_encrypt_x4_blocks_done\n\t"
5994
5995        /* Calculate H^[1-4] - GMULT partials */
5996        VMV_V_V(REG_V21, REG_V19)
5997        VMV_V_V(REG_V22, REG_V19)
5998        /* Multiply H * H => H^2 */
5999        VGMUL_VV(REG_V21, REG_V19)
6000        VMV_V_V(REG_V23, REG_V21)
6001        /* Multiply H * H => H^3 */
6002        VGMUL_VV(REG_V22, REG_V21)
6003        /* Multiply H^2 * H^2 => H^4 */
6004        VGMUL_VV(REG_V23, REG_V21)
6005
6006      "L_aes_gcm_192_encrypt_x4_block_loop:\n\t"
6007        /* Calculate next 4 counters (+1-4) */
6008#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
6009        VMV_V_V(REG_V24, REG_V16)
6010        VMV_V_V(REG_V25, REG_V16)
6011        VMV_V_V(REG_V26, REG_V16)
6012        VMV_V_V(REG_V27, REG_V16)
6013        VADD_VI(REG_V28, REG_V20, 1)
6014        VADD_VI(REG_V29, REG_V20, 2)
6015        VADD_VI(REG_V30, REG_V20, 3)
6016        VADD_VI(REG_V20, REG_V20, 4)
6017#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
6018        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
6019        VRGATHER_VV(REG_V17, REG_V15, REG_V28)
6020        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
6021        VMV_V_V(REG_V28, REG_V17)
6022#else
6023        VREV8(REG_V28, REG_V28)
6024#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
6025#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
6026        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
6027        VRGATHER_VV(REG_V17, REG_V15, REG_V29)
6028        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
6029        VMV_V_V(REG_V29, REG_V17)
6030#else
6031        VREV8(REG_V29, REG_V29)
6032#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
6033#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
6034        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
6035        VRGATHER_VV(REG_V17, REG_V15, REG_V30)
6036        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
6037        VMV_V_V(REG_V30, REG_V17)
6038#else
6039        VREV8(REG_V30, REG_V30)
6040#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
6041#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
6042        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
6043        VRGATHER_VV(REG_V31, REG_V15, REG_V20)
6044        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
6045#else
6046        VREV8(REG_V31, REG_V20)
6047#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
6048#else
6049        "addi       t0, t3, 1\n\t"
6050        VMV_V_V(REG_V24, REG_V16)
6051        "addi       t1, t3, 2\n\t"
6052        VMV_V_V(REG_V25, REG_V16)
6053        "addi       t2, t3, 3\n\t"
6054        VMV_V_V(REG_V26, REG_V16)
6055        "slli       t0, t0, 32\n\t"
6056        VMV_V_V(REG_V27, REG_V16)
6057        "slli       t1, t1, 32\n\t"
6058        "slli       t2, t2, 32\n\t"
6059        REV8(REG_T0, REG_T0)
6060        REV8(REG_T1, REG_T1)
6061        REV8(REG_T2, REG_T2)
6062        "addi       t3, t3, 4\n\t"
6063        VMV_V_X(REG_V28, REG_T0)
6064        "slli       t0, t3, 32\n\t"
6065        VMV_V_X(REG_V29, REG_T1)
6066        REV8(REG_T0, REG_T0)
6067        VMV_V_X(REG_V30, REG_T2)
6068        VMV_V_X(REG_V31, REG_T0)
6069#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
6070        VSLIDEUP_VI(REG_V24, REG_V28, 3)
6071        VSLIDEUP_VI(REG_V25, REG_V29, 3)
6072        VSLIDEUP_VI(REG_V26, REG_V30, 3)
6073        VSLIDEUP_VI(REG_V27, REG_V31, 3)
6074
6075        VAESZ_VS(REG_V24, REG_V0)
6076        VAESZ_VS(REG_V25, REG_V0)
6077        VAESZ_VS(REG_V26, REG_V0)
6078        VAESZ_VS(REG_V27, REG_V0)
6079        VAESEM_VS(REG_V24, REG_V1)
6080        VAESEM_VS(REG_V24, REG_V2)
6081        VAESEM_VS(REG_V24, REG_V3)
6082        VAESEM_VS(REG_V24, REG_V4)
6083        VAESEM_VS(REG_V24, REG_V5)
6084        VAESEM_VS(REG_V24, REG_V6)
6085        VAESEM_VS(REG_V24, REG_V7)
6086        VAESEM_VS(REG_V24, REG_V8)
6087        VAESEM_VS(REG_V24, REG_V9)
6088        VAESEM_VS(REG_V24, REG_V10)
6089        VAESEM_VS(REG_V24, REG_V11)
6090        VAESEM_VS(REG_V25, REG_V1)
6091        VAESEM_VS(REG_V25, REG_V2)
6092        VAESEM_VS(REG_V25, REG_V3)
6093        VAESEM_VS(REG_V25, REG_V4)
6094        VAESEM_VS(REG_V25, REG_V5)
6095        VAESEM_VS(REG_V25, REG_V6)
6096        VAESEM_VS(REG_V25, REG_V7)
6097        VAESEM_VS(REG_V25, REG_V8)
6098        VAESEM_VS(REG_V25, REG_V9)
6099        VAESEM_VS(REG_V25, REG_V10)
6100        VAESEM_VS(REG_V25, REG_V11)
6101        VAESEM_VS(REG_V26, REG_V1)
6102        VAESEM_VS(REG_V26, REG_V2)
6103        VAESEM_VS(REG_V26, REG_V3)
6104        VAESEM_VS(REG_V26, REG_V4)
6105        VAESEM_VS(REG_V26, REG_V5)
6106        VAESEM_VS(REG_V26, REG_V6)
6107        VAESEM_VS(REG_V26, REG_V7)
6108        VAESEM_VS(REG_V26, REG_V8)
6109        VAESEM_VS(REG_V26, REG_V9)
6110        VAESEM_VS(REG_V26, REG_V10)
6111        VAESEM_VS(REG_V26, REG_V11)
6112        VAESEM_VS(REG_V27, REG_V1)
6113        VAESEM_VS(REG_V27, REG_V2)
6114        VAESEM_VS(REG_V27, REG_V3)
6115        VAESEM_VS(REG_V27, REG_V4)
6116        VAESEM_VS(REG_V27, REG_V5)
6117        VAESEM_VS(REG_V27, REG_V6)
6118        VAESEM_VS(REG_V27, REG_V7)
6119        VAESEM_VS(REG_V27, REG_V8)
6120        VAESEM_VS(REG_V27, REG_V9)
6121        VAESEM_VS(REG_V27, REG_V10)
6122        VAESEM_VS(REG_V27, REG_V11)
6123        VAESEF_VS(REG_V24, REG_V12)
6124        VAESEF_VS(REG_V25, REG_V12)
6125        VAESEF_VS(REG_V26, REG_V12)
6126        VAESEF_VS(REG_V27, REG_V12)
6127
6128        /* Load input. */
6129        "mv        t0, %[in]\n\t"
6130        VL4RE32_V(REG_V28, REG_T0)
6131        VXOR_VV(REG_V28, REG_V24, REG_V28)
6132        VXOR_VV(REG_V29, REG_V25, REG_V29)
6133        VXOR_VV(REG_V30, REG_V26, REG_V30)
6134        VXOR_VV(REG_V31, REG_V27, REG_V31)
6135        /* Store output. */
6136        "mv         t0, %[out]\n\t"
6137        VS4R_V(REG_V28, REG_T0)
6138        VGMUL_VV(REG_V28, REG_V23)
6139        VGMUL_VV(REG_V29, REG_V22)
6140        VGMUL_VV(REG_V30, REG_V21)
6141        VGMUL_VV(REG_V31, REG_V19)
6142        VXOR_VV(REG_V18, REG_V18, REG_V28)
6143        VXOR_VV(REG_V18, REG_V18, REG_V29)
6144        VXOR_VV(REG_V18, REG_V18, REG_V30)
6145        VXOR_VV(REG_V18, REG_V18, REG_V31)
6146        "addi        %[in], %[in], 64\n\t"
6147        "addi        %[out], %[out], 64\n\t"
6148        /* Loop if more elements to process. */
6149        "addi       t4, t4, -1\n\t"
6150        "bnez       t4, L_aes_gcm_192_encrypt_x4_block_loop\n\t"
6151        "andi       %[sz], %[sz], 0x3f\n\t"
6152
6153      "L_aes_gcm_192_encrypt_x4_blocks_done:\n\t"
6154        "srli       t2, %[sz], 4\n\t"
6155        "beqz       t2, L_aes_gcm_192_encrypt_blocks_done\n\t"
6156
6157      "L_aes_gcm_192_encrypt_block_loop:\n\t"
6158#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
6159        VADD_VI(REG_V20, REG_V20, 1)
6160#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
6161        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
6162        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
6163        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
6164#else
6165        VREV8(REG_V17, REG_V20)
6166#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
6167        VMV_V_V(REG_V27, REG_V16)
6168        VSLIDEUP_VI(REG_V27, REG_V17, 3)
6169#else
6170        "addi       t3, t3, 1\n\t"
6171        "slli       t0, t3, 32\n\t"
6172        REV8(REG_T0, REG_T0)
6173        VMV_V_X(REG_V17, REG_T0)
6174        VMV_V_V(REG_V27, REG_V16)
6175        VSLIDEUP_VI(REG_V27, REG_V17, 3)
6176#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
6177
6178        VAESZ_VS(REG_V27, REG_V0)
6179        VAESEM_VS(REG_V27, REG_V1)
6180        VAESEM_VS(REG_V27, REG_V2)
6181        VAESEM_VS(REG_V27, REG_V3)
6182        VAESEM_VS(REG_V27, REG_V4)
6183        VAESEM_VS(REG_V27, REG_V5)
6184        VAESEM_VS(REG_V27, REG_V6)
6185        VAESEM_VS(REG_V27, REG_V7)
6186        VAESEM_VS(REG_V27, REG_V8)
6187        VAESEM_VS(REG_V27, REG_V9)
6188        VAESEM_VS(REG_V27, REG_V10)
6189        VAESEM_VS(REG_V27, REG_V11)
6190        VAESEF_VS(REG_V27, REG_V12)
6191
6192        /* Load input. */
6193        "mv         t0, %[in]\n\t"
6194        VL1RE32_V(REG_V17, REG_T0)
6195        VXOR_VV(REG_V27, REG_V27, REG_V17)
6196        VGHSH_VV(REG_V18, REG_V27, REG_V19)
6197        /* Store output. */
6198        "mv         t0, %[out]\n\t"
6199        VS1R_V(REG_V27, REG_T0)
6200
6201        "addi        %[in], %[in], 16\n\t"
6202        "addi        %[out], %[out], 16\n\t"
6203        /* Loop if more elements to process. */
6204        "addi       t2, t2, -1\n\t"
6205        "bnez       t2, L_aes_gcm_192_encrypt_block_loop\n\t"
6206
6207      "L_aes_gcm_192_encrypt_blocks_done:\n\t"
6208        "andi       t2, %[sz], 0xf\n\t"
6209        "beqz       t2, L_aes_gcm_192_encrypt_done\n\t"
6210
6211        VXOR_VV(REG_V17, REG_V17, REG_V17)
6212        "mv         t0, %[scratch]\n\t"
6213        VS1R_V(REG_V17, REG_T0)
6214        "mv         t1, t2\n\t"
6215      "L_aes_gcm_192_encrypt_load_byte:\n\t"
6216        "lb         t0, (%[in])\n\t"
6217        "sb         t0, (%[scratch])\n\t"
6218        "addi       %[in], %[in], 1\n\t"
6219        "addi       %[scratch], %[scratch], 1\n\t"
6220        "addi       t1, t1, -1\n\t"
6221        "bnez       t1, L_aes_gcm_192_encrypt_load_byte\n\t"
6222        "sub        %[scratch], %[scratch], t2\n\t"
6223
6224        /* Encrypt counter for partial block. */
6225#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
6226        VADD_VI(REG_V20, REG_V20, 1)
6227#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
6228        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
6229        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
6230        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
6231#else
6232        VREV8(REG_V17, REG_V20)
6233#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
6234        VMV_V_V(REG_V27, REG_V16)
6235        VSLIDEUP_VI(REG_V27, REG_V17, 3)
6236#else
6237        "addi       t3, t3, 1\n\t"
6238        "slli       t0, t3, 32\n\t"
6239        REV8(REG_T0, REG_T0)
6240        VMV_V_X(REG_V17, REG_T0)
6241        VMV_V_V(REG_V27, REG_V16)
6242        VSLIDEUP_VI(REG_V27, REG_V17, 3)
6243#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
6244
6245        VAESZ_VS(REG_V27, REG_V0)
6246        VAESEM_VS(REG_V27, REG_V1)
6247        VAESEM_VS(REG_V27, REG_V2)
6248        VAESEM_VS(REG_V27, REG_V3)
6249        VAESEM_VS(REG_V27, REG_V4)
6250        VAESEM_VS(REG_V27, REG_V5)
6251        VAESEM_VS(REG_V27, REG_V6)
6252        VAESEM_VS(REG_V27, REG_V7)
6253        VAESEM_VS(REG_V27, REG_V8)
6254        VAESEM_VS(REG_V27, REG_V9)
6255        VAESEM_VS(REG_V27, REG_V10)
6256        VAESEM_VS(REG_V27, REG_V11)
6257        VAESEF_VS(REG_V27, REG_V12)
6258
6259        /* Load scratch. */
6260        "mv         t0, %[scratch]\n\t"
6261        VL1RE32_V(REG_V17, REG_T0)
6262        VXOR_VV(REG_V27, REG_V27, REG_V17)
6263        /* Store scratch. */
6264        VS1R_V(REG_V27, REG_T0)
6265        "mv         t1, t2\n\t"
6266      "L_aes_gcm_192_encrypt_store_byte:\n\t"
6267        "lb         t0, (%[scratch])\n\t"
6268        "sb         t0, (%[out])\n\t"
6269        "addi       %[scratch], %[scratch], 1\n\t"
6270        "addi       %[out], %[out], 1\n\t"
6271        "addi       t1, t1, -1\n\t"
6272        "bnez       t1, L_aes_gcm_192_encrypt_store_byte\n\t"
6273        "li         t1, 16\n\t"
6274        "sub        t1, t1, t2\n\t"
6275      "L_aes_gcm_192_encrypt_zero_byte:\n\t"
6276        "sb         x0, (%[scratch])\n\t"
6277        "addi       %[scratch], %[scratch], 1\n\t"
6278        "addi       t1, t1, -1\n\t"
6279        "bnez       t1, L_aes_gcm_192_encrypt_zero_byte\n\t"
6280        "addi       %[scratch], %[scratch], -16\n\t"
6281        "mv         t0, %[scratch]\n\t"
6282        VL1RE32_V(REG_V17, REG_T0)
6283        VGHSH_VV(REG_V18, REG_V17, REG_V19)
6284
6285      "L_aes_gcm_192_encrypt_done:\n\t"
6286
6287        /* Hash in the lengths of A and C in bits */
6288#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
6289        /* aSz is only 32-bits */
6290        /* Multiply by 8 do get size in bits. */
6291        "slli       %[aSz], %[aSz], 3\n\t"
6292        "srli       t0, %[aSz], 32\n\t"
6293        "srli       t1, %[aSz], 24\n\t"
6294        "srli       t2, %[aSz], 16\n\t"
6295        "srli       t3, %[aSz], 8\n\t"
6296        /* Top 3 bytes are 0. */
6297        "sh         x0    , 0(%[scratch])\n\t"
6298        "sb         x0    , 2(%[scratch])\n\t"
6299        "sb         t0    , 3(%[scratch])\n\t"
6300        "sb         t1    , 4(%[scratch])\n\t"
6301        "sb         t2    , 5(%[scratch])\n\t"
6302        "sb         t3    , 6(%[scratch])\n\t"
6303        "sb         %[aSz], 7(%[scratch])\n\t"
6304        /* sz is only 32-bits */
6305        /* Multiply by 8 do get size in bits. */
6306        "slli       %[sz], %[sz], 3\n\t"
6307        "srli       t0, %[sz], 32\n\t"
6308        "srli       t1, %[sz], 24\n\t"
6309        "srli       t2, %[sz], 16\n\t"
6310        "srli       t3, %[sz], 8\n\t"
6311        /* Top 3 bytes are 0. */
6312        "sh         x0   ,  8(%[scratch])\n\t"
6313        "sb         x0   , 10(%[scratch])\n\t"
6314        "sb         t0   , 11(%[scratch])\n\t"
6315        "sb         t1   , 12(%[scratch])\n\t"
6316        "sb         t2   , 13(%[scratch])\n\t"
6317        "sb         t3   , 14(%[scratch])\n\t"
6318        "sb         %[sz], 15(%[scratch])\n\t"
6319#else
6320        "slli       t0, %[aSz], 3\n\t"
6321        REV8(REG_T0, REG_T0)
6322        "sd         t0, 0(%[scratch])\n\t"
6323        "slli       t0, %[sz], 3\n\t"
6324        REV8(REG_T0, REG_T0)
6325        "sd         t0, 8(%[scratch])\n\t"
6326#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
6327        "mv         t0, %[scratch]\n\t"
6328        VL1RE32_V(REG_V17, REG_T0)
6329        VGHSH_VV(REG_V18, REG_V17, REG_V19)
6330
6331        VAESZ_VS(REG_V16, REG_V0)
6332        VAESEM_VS(REG_V16, REG_V1)
6333        VAESEM_VS(REG_V16, REG_V2)
6334        VAESEM_VS(REG_V16, REG_V3)
6335        VAESEM_VS(REG_V16, REG_V4)
6336        VAESEM_VS(REG_V16, REG_V5)
6337        VAESEM_VS(REG_V16, REG_V6)
6338        VAESEM_VS(REG_V16, REG_V7)
6339        VAESEM_VS(REG_V16, REG_V8)
6340        VAESEM_VS(REG_V16, REG_V9)
6341        VAESEM_VS(REG_V16, REG_V10)
6342        VAESEM_VS(REG_V16, REG_V11)
6343        VAESEF_VS(REG_V16, REG_V12)
6344        VXOR_VV(REG_V18, REG_V18, REG_V16)
6345
6346        "li         t1, 16\n\t"
6347        "blt        %[tagSz], t1, L_aes_gcm_192_encrypt_tag_small\n\t"
6348        "mv         t0, %[tag]\n\t"
6349        VS1R_V(REG_V18, REG_T0)
6350        "beqz       x0, L_aes_gcm_192_encrypt_tag_done\n\t"
6351      "L_aes_gcm_192_encrypt_tag_small:\n\t"
6352        "mv         t0, %[scratch]\n\t"
6353        VS1R_V(REG_V18, REG_T0)
6354        "mv         t1, %[tagSz]\n\t"
6355      "L_aes_gcm_192_encrypt_store_tag_byte:\n\t"
6356        "lb         t0, (%[scratch])\n\t"
6357        "sb         t0, (%[tag])\n\t"
6358        "addi       %[scratch], %[scratch], 1\n\t"
6359        "addi       %[tag], %[tag], 1\n\t"
6360        "addi       t1, t1, -1\n\t"
6361        "bnez       t1, L_aes_gcm_192_encrypt_store_tag_byte\n\t"
6362      "L_aes_gcm_192_encrypt_tag_done:\n\t"
6363
6364        : [out] "+r" (out), [in] "+r" (in), [key] "+r" (key),
6365          [aSz] "+r" (aadSz), [aad] "+r" (aad), [sz] "+r" (sz)
6366        : [ctr] "r" (ctr), [scratch] "r" (scratch),
6367          [h] "r" (aes->gcm.H), [tag] "r" (tag), [tagSz] "r" (tagSz)
6368#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
6369          , [rev_idx] "r" (rev_idx)
6370#endif
6371        : "memory", "t0", "t1", "t2", "t3", "t4"
6372    );
6373
6374#ifdef OPENSSL_EXTRA
6375    if ((tag != NULL) && (in != NULL) && (sz != 0)) {
6376        /* store AAD size for next call */
6377        aes->gcm.aadLen = aadSz;
6378    }
6379#endif
6380}
6381#endif /*  WOLFSSL_AES_192 */
6382
6383#ifdef WOLFSSL_AES_256
6384/* Encrypt data using AES-256-GCM.
6385 *
6386 * @param [in]  aes      AES object.
6387 * @param [out] out      Encrypted data.
6388 * @param [in]  in       Data to encrypt and GHASH.
6389 * @param [in]  sz       Number of bytes of data.
6390 * @param [in]  nonce    Nonce used to calculate first IV.
6391 * @param [in]  nonceSz  Length of nonce in bytes.
6392 * @param [out] tag      Authentication tag.
6393 * @param [in]  tagSz    Length of authentication tag in bytes.
6394 * @param [in]  aad      Additional Authentication Data (AAD).
6395 * @param [in]  aadSz    Length of AAD in bytes.
6396 */
6397static void Aes256GcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz,
6398    const byte* nonce, word32 nonceSz, byte* tag, word32 tagSz,
6399    const byte* aad, word32 aadSz)
6400{
6401    byte counter[WC_AES_BLOCK_SIZE];
6402    byte scratch[WC_AES_BLOCK_SIZE];
6403    /* Noticed different optimization levels treated head of array different.
6404     * Some cases was stack pointer plus offset others was a register containing
6405     * address. To make uniform for passing in to inline assembly code am using
6406     * pointers to the head of each local array.
6407     */
6408    byte* ctr  = counter;
6409    byte* key = (byte*)aes->key;
6410
6411    XMEMSET(counter, 0, WC_AES_BLOCK_SIZE);
6412    if (nonceSz == GCM_NONCE_MID_SZ) {
6413        XMEMCPY(counter, nonce, GCM_NONCE_MID_SZ);
6414        counter[WC_AES_BLOCK_SIZE - 1] = 1;
6415    }
6416    else {
6417#ifdef OPENSSL_EXTRA
6418        word32 aadTemp = aes->gcm.aadLen;
6419        aes->gcm.aadLen = 0;
6420#endif
6421        GHASH(&aes->gcm, NULL, 0, nonce, nonceSz, counter, WC_AES_BLOCK_SIZE);
6422#ifdef OPENSSL_EXTRA
6423        aes->gcm.aadLen = aadTemp;
6424#endif
6425    }
6426
6427    __asm__ __volatile__ (
6428        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
6429
6430        /* X=0, get H */
6431        VXOR_VV(REG_V18, REG_V18, REG_V18)
6432        "mv         t0, %[h]\n\t"
6433        VL1RE32_V(REG_V19, REG_T0)
6434
6435        /* Hash in AAD, the Additional Authentication Data */
6436        "beqz       %[aSz], L_aes_gcm_256_encrypt_ghash_aad_done\n\t"
6437        "beqz       %[aad], L_aes_gcm_256_encrypt_ghash_aad_done\n\t"
6438
6439        "srli       t1, %[aSz], 4\n\t"
6440        "beqz       t1, L_aes_gcm_256_encrypt_ghash_aad_blocks_done\n\t"
6441
6442      "L_aes_gcm_256_encrypt_ghash_aad_loop:\n\t"
6443        "mv         t0, %[aad]\n\t"
6444        VL1RE32_V(REG_V17, REG_T0)
6445        VGHSH_VV(REG_V18, REG_V17, REG_V19)
6446        "addi       %[aad], %[aad], 16\n\t"
6447        "addi       t1, t1, -1\n\t"
6448        "bnez       t1, L_aes_gcm_256_encrypt_ghash_aad_loop\n\t"
6449      "L_aes_gcm_256_encrypt_ghash_aad_blocks_done:\n\t"
6450        "andi       t1, %[aSz], 0xf\n\t"
6451        "beqz       t1, L_aes_gcm_256_encrypt_ghash_aad_done\n\t"
6452        VXOR_VV(REG_V17, REG_V17, REG_V17)
6453        "mv         t0, %[scratch]\n\t"
6454        VS1R_V(REG_V17, REG_T0)
6455        "mv         t2, t1\n\t"
6456      "L_aes_gcm_256_encrypt_ghash_aad_load_byte:\n\t"
6457        "lb         t0, (%[aad])\n\t"
6458        "sb         t0, (%[scratch])\n\t"
6459        "addi       %[aad], %[aad], 1\n\t"
6460        "addi       %[scratch], %[scratch], 1\n\t"
6461        "addi       t2, t2, -1\n\t"
6462        "bnez       t2, L_aes_gcm_256_encrypt_ghash_aad_load_byte\n\t"
6463        "sub        %[scratch], %[scratch], t1\n\t"
6464        "mv         t0, %[scratch]\n\t"
6465        VL1RE32_V(REG_V17, REG_T0)
6466        VGHSH_VV(REG_V18, REG_V17, REG_V19)
6467      "L_aes_gcm_256_encrypt_ghash_aad_done:\n\t"
6468        /* Done Hash in AAD */
6469
6470#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
6471        "mv         t0, %[rev_idx]\n\t"
6472        VL1RE32_V(REG_V15, REG_T0)
6473#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
6474        /* Load the counter. */
6475        "mv         t0, %[ctr]\n\t"
6476        VL1RE32_V(REG_V16, REG_T0)
6477#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
6478        VSLIDEDOWN_VI(REG_V20, REG_V16, 3)
6479#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
6480        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
6481        VRGATHER_VV(REG_V21, REG_V15, REG_V20)
6482        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
6483        VMV_V_V(REG_V20, REG_V21)
6484#else
6485        VREV8(REG_V20, REG_V20)
6486#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
6487#else
6488        "lw         t3, 12(%[ctr])\n\t"
6489        "slli       t3, t3, 32\n\t"
6490        REV8(REG_T3, REG_T3)
6491#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
6492
6493        /* Load key[0..7]. */
6494        "mv         t0, %[key]\n\t"
6495        VL8RE32_V(REG_V0, REG_T0)
6496        /* Load key[8..11]. */
6497        "addi       t0, t0, 128\n\t"
6498        VL4RE32_V(REG_V8, REG_T0)
6499        /* Load key[12..13]. */
6500        "addi       t0, t0, 64\n\t"
6501        VL2RE32_V(REG_V12, REG_T0)
6502        /* Load last round's key */
6503        "addi       t0, %[key], 224\n\t"
6504        VL1RE32_V(REG_V14, REG_T0)
6505
6506        "beqz       %[sz], L_aes_gcm_256_encrypt_blocks_done\n\t"
6507        "srli       t4, %[sz], 6\n\t"
6508        "beqz       t4, L_aes_gcm_256_encrypt_x4_blocks_done\n\t"
6509
6510        /* Calculate H^[1-4] - GMULT partials */
6511        VMV_V_V(REG_V21, REG_V19)
6512        VMV_V_V(REG_V22, REG_V19)
6513        /* Multiply H * H => H^2 */
6514        VGMUL_VV(REG_V21, REG_V19)
6515        VMV_V_V(REG_V23, REG_V21)
6516        /* Multiply H * H => H^3 */
6517        VGMUL_VV(REG_V22, REG_V21)
6518        /* Multiply H^2 * H^2 => H^4 */
6519        VGMUL_VV(REG_V23, REG_V21)
6520
6521      "L_aes_gcm_256_encrypt_x4_block_loop:\n\t"
6522        /* Calculate next 4 counters (+1-4) */
6523#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
6524        VMV_V_V(REG_V24, REG_V16)
6525        VMV_V_V(REG_V25, REG_V16)
6526        VMV_V_V(REG_V26, REG_V16)
6527        VMV_V_V(REG_V27, REG_V16)
6528        VADD_VI(REG_V28, REG_V20, 1)
6529        VADD_VI(REG_V29, REG_V20, 2)
6530        VADD_VI(REG_V30, REG_V20, 3)
6531        VADD_VI(REG_V20, REG_V20, 4)
6532#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
6533        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
6534        VRGATHER_VV(REG_V17, REG_V15, REG_V28)
6535        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
6536        VMV_V_V(REG_V28, REG_V17)
6537#else
6538        VREV8(REG_V28, REG_V28)
6539#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
6540#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
6541        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
6542        VRGATHER_VV(REG_V17, REG_V15, REG_V29)
6543        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
6544        VMV_V_V(REG_V29, REG_V17)
6545#else
6546        VREV8(REG_V29, REG_V29)
6547#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
6548#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
6549        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
6550        VRGATHER_VV(REG_V17, REG_V15, REG_V30)
6551        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
6552        VMV_V_V(REG_V30, REG_V17)
6553#else
6554        VREV8(REG_V30, REG_V30)
6555#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
6556#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
6557        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
6558        VRGATHER_VV(REG_V31, REG_V15, REG_V20)
6559        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
6560#else
6561        VREV8(REG_V31, REG_V20)
6562#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
6563#else
6564        "addi       t0, t3, 1\n\t"
6565        VMV_V_V(REG_V24, REG_V16)
6566        "addi       t1, t3, 2\n\t"
6567        VMV_V_V(REG_V25, REG_V16)
6568        "addi       t2, t3, 3\n\t"
6569        VMV_V_V(REG_V26, REG_V16)
6570        "slli       t0, t0, 32\n\t"
6571        VMV_V_V(REG_V27, REG_V16)
6572        "slli       t1, t1, 32\n\t"
6573        "slli       t2, t2, 32\n\t"
6574        REV8(REG_T0, REG_T0)
6575        REV8(REG_T1, REG_T1)
6576        REV8(REG_T2, REG_T2)
6577        "addi       t3, t3, 4\n\t"
6578        VMV_V_X(REG_V28, REG_T0)
6579        "slli       t0, t3, 32\n\t"
6580        VMV_V_X(REG_V29, REG_T1)
6581        REV8(REG_T0, REG_T0)
6582        VMV_V_X(REG_V30, REG_T2)
6583        VMV_V_X(REG_V31, REG_T0)
6584#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
6585        VSLIDEUP_VI(REG_V24, REG_V28, 3)
6586        VSLIDEUP_VI(REG_V25, REG_V29, 3)
6587        VSLIDEUP_VI(REG_V26, REG_V30, 3)
6588        VSLIDEUP_VI(REG_V27, REG_V31, 3)
6589
6590        VAESZ_VS(REG_V24, REG_V0)
6591        VAESZ_VS(REG_V25, REG_V0)
6592        VAESZ_VS(REG_V26, REG_V0)
6593        VAESZ_VS(REG_V27, REG_V0)
6594        VAESEM_VS(REG_V24, REG_V1)
6595        VAESEM_VS(REG_V24, REG_V2)
6596        VAESEM_VS(REG_V24, REG_V3)
6597        VAESEM_VS(REG_V24, REG_V4)
6598        VAESEM_VS(REG_V24, REG_V5)
6599        VAESEM_VS(REG_V24, REG_V6)
6600        VAESEM_VS(REG_V24, REG_V7)
6601        VAESEM_VS(REG_V24, REG_V8)
6602        VAESEM_VS(REG_V24, REG_V9)
6603        VAESEM_VS(REG_V24, REG_V10)
6604        VAESEM_VS(REG_V24, REG_V11)
6605        VAESEM_VS(REG_V24, REG_V12)
6606        VAESEM_VS(REG_V24, REG_V13)
6607        VAESEM_VS(REG_V25, REG_V1)
6608        VAESEM_VS(REG_V25, REG_V2)
6609        VAESEM_VS(REG_V25, REG_V3)
6610        VAESEM_VS(REG_V25, REG_V4)
6611        VAESEM_VS(REG_V25, REG_V5)
6612        VAESEM_VS(REG_V25, REG_V6)
6613        VAESEM_VS(REG_V25, REG_V7)
6614        VAESEM_VS(REG_V25, REG_V8)
6615        VAESEM_VS(REG_V25, REG_V9)
6616        VAESEM_VS(REG_V25, REG_V10)
6617        VAESEM_VS(REG_V25, REG_V11)
6618        VAESEM_VS(REG_V25, REG_V12)
6619        VAESEM_VS(REG_V25, REG_V13)
6620        VAESEM_VS(REG_V26, REG_V1)
6621        VAESEM_VS(REG_V26, REG_V2)
6622        VAESEM_VS(REG_V26, REG_V3)
6623        VAESEM_VS(REG_V26, REG_V4)
6624        VAESEM_VS(REG_V26, REG_V5)
6625        VAESEM_VS(REG_V26, REG_V6)
6626        VAESEM_VS(REG_V26, REG_V7)
6627        VAESEM_VS(REG_V26, REG_V8)
6628        VAESEM_VS(REG_V26, REG_V9)
6629        VAESEM_VS(REG_V26, REG_V10)
6630        VAESEM_VS(REG_V26, REG_V11)
6631        VAESEM_VS(REG_V26, REG_V12)
6632        VAESEM_VS(REG_V26, REG_V13)
6633        VAESEM_VS(REG_V27, REG_V1)
6634        VAESEM_VS(REG_V27, REG_V2)
6635        VAESEM_VS(REG_V27, REG_V3)
6636        VAESEM_VS(REG_V27, REG_V4)
6637        VAESEM_VS(REG_V27, REG_V5)
6638        VAESEM_VS(REG_V27, REG_V6)
6639        VAESEM_VS(REG_V27, REG_V7)
6640        VAESEM_VS(REG_V27, REG_V8)
6641        VAESEM_VS(REG_V27, REG_V9)
6642        VAESEM_VS(REG_V27, REG_V10)
6643        VAESEM_VS(REG_V27, REG_V11)
6644        VAESEM_VS(REG_V27, REG_V12)
6645        VAESEM_VS(REG_V27, REG_V13)
6646        VAESEF_VS(REG_V24, REG_V14)
6647        VAESEF_VS(REG_V25, REG_V14)
6648        VAESEF_VS(REG_V26, REG_V14)
6649        VAESEF_VS(REG_V27, REG_V14)
6650
6651        /* Load input. */
6652        "mv        t0, %[in]\n\t"
6653        VL4RE32_V(REG_V28, REG_T0)
6654        VXOR_VV(REG_V28, REG_V24, REG_V28)
6655        VXOR_VV(REG_V29, REG_V25, REG_V29)
6656        VXOR_VV(REG_V30, REG_V26, REG_V30)
6657        VXOR_VV(REG_V31, REG_V27, REG_V31)
6658        /* Store output. */
6659        "mv         t0, %[out]\n\t"
6660        VS4R_V(REG_V28, REG_T0)
6661        VGMUL_VV(REG_V28, REG_V23)
6662        VGMUL_VV(REG_V29, REG_V22)
6663        VGMUL_VV(REG_V30, REG_V21)
6664        VGMUL_VV(REG_V31, REG_V19)
6665        VXOR_VV(REG_V18, REG_V18, REG_V28)
6666        VXOR_VV(REG_V18, REG_V18, REG_V29)
6667        VXOR_VV(REG_V18, REG_V18, REG_V30)
6668        VXOR_VV(REG_V18, REG_V18, REG_V31)
6669        "addi        %[in], %[in], 64\n\t"
6670        "addi        %[out], %[out], 64\n\t"
6671        /* Loop if more elements to process. */
6672        "addi       t4, t4, -1\n\t"
6673        "bnez       t4, L_aes_gcm_256_encrypt_x4_block_loop\n\t"
6674        "andi       %[sz], %[sz], 0x3f\n\t"
6675
6676      "L_aes_gcm_256_encrypt_x4_blocks_done:\n\t"
6677        "srli       t2, %[sz], 4\n\t"
6678        "beqz       t2, L_aes_gcm_256_encrypt_blocks_done\n\t"
6679
6680      "L_aes_gcm_256_encrypt_block_loop:\n\t"
6681#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
6682        VADD_VI(REG_V20, REG_V20, 1)
6683#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
6684        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
6685        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
6686        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
6687#else
6688        VREV8(REG_V17, REG_V20)
6689#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
6690        VMV_V_V(REG_V27, REG_V16)
6691        VSLIDEUP_VI(REG_V27, REG_V17, 3)
6692#else
6693        "addi       t3, t3, 1\n\t"
6694        "slli       t0, t3, 32\n\t"
6695        REV8(REG_T0, REG_T0)
6696        VMV_V_X(REG_V17, REG_T0)
6697        VMV_V_V(REG_V27, REG_V16)
6698        VSLIDEUP_VI(REG_V27, REG_V17, 3)
6699#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
6700
6701        VAESZ_VS(REG_V27, REG_V0)
6702        VAESEM_VS(REG_V27, REG_V1)
6703        VAESEM_VS(REG_V27, REG_V2)
6704        VAESEM_VS(REG_V27, REG_V3)
6705        VAESEM_VS(REG_V27, REG_V4)
6706        VAESEM_VS(REG_V27, REG_V5)
6707        VAESEM_VS(REG_V27, REG_V6)
6708        VAESEM_VS(REG_V27, REG_V7)
6709        VAESEM_VS(REG_V27, REG_V8)
6710        VAESEM_VS(REG_V27, REG_V9)
6711        VAESEM_VS(REG_V27, REG_V10)
6712        VAESEM_VS(REG_V27, REG_V11)
6713        VAESEM_VS(REG_V27, REG_V12)
6714        VAESEM_VS(REG_V27, REG_V13)
6715        VAESEF_VS(REG_V27, REG_V14)
6716
6717        /* Load input. */
6718        "mv         t0, %[in]\n\t"
6719        VL1RE32_V(REG_V17, REG_T0)
6720        VXOR_VV(REG_V27, REG_V27, REG_V17)
6721        VGHSH_VV(REG_V18, REG_V27, REG_V19)
6722        /* Store output. */
6723        "mv         t0, %[out]\n\t"
6724        VS1R_V(REG_V27, REG_T0)
6725
6726        "addi        %[in], %[in], 16\n\t"
6727        "addi        %[out], %[out], 16\n\t"
6728        /* Loop if more elements to process. */
6729        "addi       t2, t2, -1\n\t"
6730        "bnez       t2, L_aes_gcm_256_encrypt_block_loop\n\t"
6731
6732      "L_aes_gcm_256_encrypt_blocks_done:\n\t"
6733        "andi       t2, %[sz], 0xf\n\t"
6734        "beqz       t2, L_aes_gcm_256_encrypt_done\n\t"
6735
6736        VXOR_VV(REG_V17, REG_V17, REG_V17)
6737        "mv         t0, %[scratch]\n\t"
6738        VS1R_V(REG_V17, REG_T0)
6739        "mv         t1, t2\n\t"
6740      "L_aes_gcm_256_encrypt_load_byte:\n\t"
6741        "lb         t0, (%[in])\n\t"
6742        "sb         t0, (%[scratch])\n\t"
6743        "addi       %[in], %[in], 1\n\t"
6744        "addi       %[scratch], %[scratch], 1\n\t"
6745        "addi       t1, t1, -1\n\t"
6746        "bnez       t1, L_aes_gcm_256_encrypt_load_byte\n\t"
6747        "sub        %[scratch], %[scratch], t2\n\t"
6748
6749        /* Encrypt counter for partial block. */
6750#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
6751        VADD_VI(REG_V20, REG_V20, 1)
6752#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
6753        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
6754        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
6755        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
6756#else
6757        VREV8(REG_V17, REG_V20)
6758#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
6759        VMV_V_V(REG_V27, REG_V16)
6760        VSLIDEUP_VI(REG_V27, REG_V17, 3)
6761#else
6762        "addi       t3, t3, 1\n\t"
6763        "slli       t0, t3, 32\n\t"
6764        REV8(REG_T0, REG_T0)
6765        VMV_V_X(REG_V17, REG_T0)
6766        VMV_V_V(REG_V27, REG_V16)
6767        VSLIDEUP_VI(REG_V27, REG_V17, 3)
6768#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
6769
6770        VAESZ_VS(REG_V27, REG_V0)
6771        VAESEM_VS(REG_V27, REG_V1)
6772        VAESEM_VS(REG_V27, REG_V2)
6773        VAESEM_VS(REG_V27, REG_V3)
6774        VAESEM_VS(REG_V27, REG_V4)
6775        VAESEM_VS(REG_V27, REG_V5)
6776        VAESEM_VS(REG_V27, REG_V6)
6777        VAESEM_VS(REG_V27, REG_V7)
6778        VAESEM_VS(REG_V27, REG_V8)
6779        VAESEM_VS(REG_V27, REG_V9)
6780        VAESEM_VS(REG_V27, REG_V10)
6781        VAESEM_VS(REG_V27, REG_V11)
6782        VAESEM_VS(REG_V27, REG_V12)
6783        VAESEM_VS(REG_V27, REG_V13)
6784        VAESEF_VS(REG_V27, REG_V14)
6785
6786        /* Load scratch. */
6787        "mv         t0, %[scratch]\n\t"
6788        VL1RE32_V(REG_V17, REG_T0)
6789        VXOR_VV(REG_V27, REG_V27, REG_V17)
6790        /* Store scratch. */
6791        VS1R_V(REG_V27, REG_T0)
6792        "mv         t1, t2\n\t"
6793      "L_aes_gcm_256_encrypt_store_byte:\n\t"
6794        "lb         t0, (%[scratch])\n\t"
6795        "sb         t0, (%[out])\n\t"
6796        "addi       %[scratch], %[scratch], 1\n\t"
6797        "addi       %[out], %[out], 1\n\t"
6798        "addi       t1, t1, -1\n\t"
6799        "bnez       t1, L_aes_gcm_256_encrypt_store_byte\n\t"
6800        "li         t1, 16\n\t"
6801        "sub        t1, t1, t2\n\t"
6802      "L_aes_gcm_256_encrypt_zero_byte:\n\t"
6803        "sb         x0, (%[scratch])\n\t"
6804        "addi       %[scratch], %[scratch], 1\n\t"
6805        "addi       t1, t1, -1\n\t"
6806        "bnez       t1, L_aes_gcm_256_encrypt_zero_byte\n\t"
6807        "addi       %[scratch], %[scratch], -16\n\t"
6808        "mv         t0, %[scratch]\n\t"
6809        VL1RE32_V(REG_V17, REG_T0)
6810        VGHSH_VV(REG_V18, REG_V17, REG_V19)
6811
6812      "L_aes_gcm_256_encrypt_done:\n\t"
6813
6814        /* Hash in the lengths of A and C in bits */
6815#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
6816        /* aSz is only 32-bits */
6817        /* Multiply by 8 do get size in bits. */
6818        "slli       %[aSz], %[aSz], 3\n\t"
6819        "srli       t0, %[aSz], 32\n\t"
6820        "srli       t1, %[aSz], 24\n\t"
6821        "srli       t2, %[aSz], 16\n\t"
6822        "srli       t3, %[aSz], 8\n\t"
6823        /* Top 3 bytes are 0. */
6824        "sh         x0    , 0(%[scratch])\n\t"
6825        "sb         x0    , 2(%[scratch])\n\t"
6826        "sb         t0    , 3(%[scratch])\n\t"
6827        "sb         t1    , 4(%[scratch])\n\t"
6828        "sb         t2    , 5(%[scratch])\n\t"
6829        "sb         t3    , 6(%[scratch])\n\t"
6830        "sb         %[aSz], 7(%[scratch])\n\t"
6831        /* sz is only 32-bits */
6832        /* Multiply by 8 do get size in bits. */
6833        "slli       %[sz], %[sz], 3\n\t"
6834        "srli       t0, %[sz], 32\n\t"
6835        "srli       t1, %[sz], 24\n\t"
6836        "srli       t2, %[sz], 16\n\t"
6837        "srli       t3, %[sz], 8\n\t"
6838        /* Top 3 bytes are 0. */
6839        "sh         x0   ,  8(%[scratch])\n\t"
6840        "sb         x0   , 10(%[scratch])\n\t"
6841        "sb         t0   , 11(%[scratch])\n\t"
6842        "sb         t1   , 12(%[scratch])\n\t"
6843        "sb         t2   , 13(%[scratch])\n\t"
6844        "sb         t3   , 14(%[scratch])\n\t"
6845        "sb         %[sz], 15(%[scratch])\n\t"
6846#else
6847        "slli       t0, %[aSz], 3\n\t"
6848        REV8(REG_T0, REG_T0)
6849        "sd         t0, 0(%[scratch])\n\t"
6850        "slli       t0, %[sz], 3\n\t"
6851        REV8(REG_T0, REG_T0)
6852        "sd         t0, 8(%[scratch])\n\t"
6853#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
6854        "mv         t0, %[scratch]\n\t"
6855        VL1RE32_V(REG_V17, REG_T0)
6856        VGHSH_VV(REG_V18, REG_V17, REG_V19)
6857
6858        VAESZ_VS(REG_V16, REG_V0)
6859        VAESEM_VS(REG_V16, REG_V1)
6860        VAESEM_VS(REG_V16, REG_V2)
6861        VAESEM_VS(REG_V16, REG_V3)
6862        VAESEM_VS(REG_V16, REG_V4)
6863        VAESEM_VS(REG_V16, REG_V5)
6864        VAESEM_VS(REG_V16, REG_V6)
6865        VAESEM_VS(REG_V16, REG_V7)
6866        VAESEM_VS(REG_V16, REG_V8)
6867        VAESEM_VS(REG_V16, REG_V9)
6868        VAESEM_VS(REG_V16, REG_V10)
6869        VAESEM_VS(REG_V16, REG_V11)
6870        VAESEM_VS(REG_V16, REG_V12)
6871        VAESEM_VS(REG_V16, REG_V13)
6872        VAESEF_VS(REG_V16, REG_V14)
6873        VXOR_VV(REG_V18, REG_V18, REG_V16)
6874
6875        "li         t1, 16\n\t"
6876        "blt        %[tagSz], t1, L_aes_gcm_256_encrypt_tag_small\n\t"
6877        "mv         t0, %[tag]\n\t"
6878        VS1R_V(REG_V18, REG_T0)
6879        "beqz       x0, L_aes_gcm_256_encrypt_tag_done\n\t"
6880      "L_aes_gcm_256_encrypt_tag_small:\n\t"
6881        "mv         t0, %[scratch]\n\t"
6882        VS1R_V(REG_V18, REG_T0)
6883        "mv         t1, %[tagSz]\n\t"
6884      "L_aes_gcm_256_encrypt_store_tag_byte:\n\t"
6885        "lb         t0, (%[scratch])\n\t"
6886        "sb         t0, (%[tag])\n\t"
6887        "addi       %[scratch], %[scratch], 1\n\t"
6888        "addi       %[tag], %[tag], 1\n\t"
6889        "addi       t1, t1, -1\n\t"
6890        "bnez       t1, L_aes_gcm_256_encrypt_store_tag_byte\n\t"
6891      "L_aes_gcm_256_encrypt_tag_done:\n\t"
6892
6893        : [out] "+r" (out), [in] "+r" (in), [key] "+r" (key),
6894          [aSz] "+r" (aadSz), [aad] "+r" (aad), [sz] "+r" (sz)
6895        : [ctr] "r" (ctr), [scratch] "r" (scratch),
6896          [h] "r" (aes->gcm.H), [tag] "r" (tag), [tagSz] "r" (tagSz)
6897#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
6898          , [rev_idx] "r" (rev_idx)
6899#endif
6900        : "memory", "t0", "t1", "t2", "t3", "t4"
6901    );
6902
6903#ifdef OPENSSL_EXTRA
6904    if ((tag != NULL) && (in != NULL) && (sz != 0)) {
6905        /* store AAD size for next call */
6906        aes->gcm.aadLen = aadSz;
6907    }
6908#endif
6909}
6910#endif /*  WOLFSSL_AES_256 */
6911
6912/* Encrypt data using AES-GCM.
6913 *
6914 * @param [in]  aes      AES object.
6915 * @param [out] out      Encrypted data.
6916 * @param [in]  in       Data to encrypt.
6917 * @param [in]  sz       Number of bytes of data.
6918 * @param [in]  nonce    Nonce used to calculate first IV.
6919 * @param [in]  nonceSz  Length of nonce in bytes.
6920 * @param [out] tag      Authentication tag.
6921 * @param [in]  tagSz    Length of authentication tag in bytes.
6922 * @param [in]  aad      Additional Authentication Data (AAD).
6923 * @param [in]  aadSz    Length of AAD in bytes.
6924 * @return  0 on success.
6925 * @return  BAD_FUNC_ARG when aes, nonce or tag is NULL.
6926 * @return  BAD_FUNC_ARG when nonceSz is zero.
6927 * @return  BAD_FUNC_ARG when aad is NULL but aadSz is not zero.
6928 * @return  BAD_FUNC_ARG when tagSz is less than WOLFSSL_MIN_AUTH_TAG_SZ or
6929 *          greater than WC_AES_BLOCK_SIZE.
6930 * @return  BAD_FUNC_ARG when sz is not zero but in or out is NULL.
6931 */
6932int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz,
6933    const byte* nonce, word32 nonceSz, byte* tag, word32 tagSz, const byte* aad,
6934    word32 aadSz)
6935{
6936    int ret = 0;
6937
6938    /* sanity checks */
6939    if ((aes == NULL) || (nonce == NULL) || (nonceSz == 0) || (tag == NULL) ||
6940            ((aad == NULL) && (aadSz > 0)) || ((sz != 0) && ((in == NULL) ||
6941            (out == NULL)))) {
6942        WOLFSSL_MSG("a NULL parameter passed in when size is larger than 0");
6943        ret = BAD_FUNC_ARG;
6944    }
6945
6946    if ((ret == 0) && ((tagSz < WOLFSSL_MIN_AUTH_TAG_SZ) ||
6947            (tagSz > WC_AES_BLOCK_SIZE))) {
6948        WOLFSSL_MSG("GcmEncrypt tagSz error");
6949        ret = BAD_FUNC_ARG;
6950    }
6951
6952    if (ret == 0) {
6953        switch (aes->rounds) {
6954        #ifdef WOLFSSL_AES_128
6955            case 10:
6956                Aes128GcmEncrypt(aes, out, in, sz, nonce, nonceSz, tag, tagSz,
6957                    aad, aadSz);
6958                break;
6959        #endif
6960        #ifdef WOLFSSL_AES_192
6961            case 12:
6962                Aes192GcmEncrypt(aes, out, in, sz, nonce, nonceSz, tag, tagSz,
6963                    aad, aadSz);
6964                break;
6965        #endif
6966        #ifdef WOLFSSL_AES_256
6967            case 14:
6968                Aes256GcmEncrypt(aes, out, in, sz, nonce, nonceSz, tag, tagSz,
6969                    aad, aadSz);
6970                break;
6971        #endif
6972            default:
6973                WOLFSSL_MSG("AES-GCM invalid round number");
6974                ret = BAD_FUNC_ARG;
6975        }
6976    }
6977
6978    return ret;
6979}
6980
6981
6982#ifdef HAVE_AES_DECRYPT
6983
6984#ifdef WOLFSSL_AES_128
6985/* Decrypt data using AES-128-GCM.
6986 *
6987 * @param [in]  aes      AES object.
6988 * @param [out] out      Decrypted data.
6989 * @param [in]  in       Data to decrypt and GHASH.
6990 * @param [in]  sz       Number of bytes of data.
6991 * @param [in]  nonce    Nonce used to calculate first IV.
6992 * @param [in]  nonceSz  Length of nonce in bytes.
6993 * @param [out] tag      Authentication tag.
6994 * @param [in]  tagSz    Length of authentication tag in bytes.
6995 * @param [in]  aad      Additional Authentication Data (AAD).
6996 * @param [in]  aadSz    Length of AAD in bytes.
6997 * @return  0 on success.
6998 * @return  AES_GCM_AUTH_E when authentication tag computed doesn't match
6999 *          tag passed in.
7000 */
7001static int Aes128GcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
7002    const byte* nonce, word32 nonceSz, const byte* tag, word32 tagSz,
7003    const byte* aad, word32 aadSz)
7004{
7005    int ret = 0;
7006    byte counter[WC_AES_BLOCK_SIZE];
7007    byte scratch[WC_AES_BLOCK_SIZE];
7008    /* Noticed different optimization levels treated head of array different.
7009     * Some cases was stack pointer plus offset others was a register containing
7010     * address. To make uniform for passing in to inline assembly code am using
7011     * pointers to the head of each local array.
7012     */
7013    byte* ctr  = counter;
7014    byte* key = (byte*)aes->key;
7015
7016    XMEMSET(counter, 0, WC_AES_BLOCK_SIZE);
7017    if (nonceSz == GCM_NONCE_MID_SZ) {
7018        XMEMCPY(counter, nonce, GCM_NONCE_MID_SZ);
7019        counter[WC_AES_BLOCK_SIZE - 1] = 1;
7020    }
7021    else {
7022#ifdef OPENSSL_EXTRA
7023        word32 aadTemp = aes->gcm.aadLen;
7024        aes->gcm.aadLen = 0;
7025#endif
7026        GHASH(&aes->gcm, NULL, 0, nonce, nonceSz, counter, WC_AES_BLOCK_SIZE);
7027#ifdef OPENSSL_EXTRA
7028        aes->gcm.aadLen = aadTemp;
7029#endif
7030    }
7031
7032    __asm__ __volatile__ (
7033        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
7034
7035        /* X=0, get H */
7036        VXOR_VV(REG_V18, REG_V18, REG_V18)
7037        "mv         t0, %[h]\n\t"
7038        VL1RE32_V(REG_V19, REG_T0)
7039
7040        /* Hash in AAD, the Additional Authentication Data */
7041        "beqz       %[aSz], L_aes_gcm_128_decrypt_ghash_aad_done\n\t"
7042        "beqz       %[aad], L_aes_gcm_128_decrypt_ghash_aad_done\n\t"
7043
7044        "srli       t1, %[aSz], 4\n\t"
7045        "beqz       t1, L_aes_gcm_128_decrypt_ghash_aad_blocks_done\n\t"
7046
7047      "L_aes_gcm_128_decrypt_ghash_aad_loop:\n\t"
7048        "mv         t0, %[aad]\n\t"
7049        VL1RE32_V(REG_V17, REG_T0)
7050        VGHSH_VV(REG_V18, REG_V17, REG_V19)
7051        "addi       %[aad], %[aad], 16\n\t"
7052        "addi       t1, t1, -1\n\t"
7053        "bnez       t1, L_aes_gcm_128_decrypt_ghash_aad_loop\n\t"
7054      "L_aes_gcm_128_decrypt_ghash_aad_blocks_done:\n\t"
7055        "andi       t1, %[aSz], 0xf\n\t"
7056        "beqz       t1, L_aes_gcm_128_decrypt_ghash_aad_done\n\t"
7057        VXOR_VV(REG_V17, REG_V17, REG_V17)
7058        "mv         t0, %[scratch]\n\t"
7059        VS1R_V(REG_V17, REG_T0)
7060        "mv         t2, t1\n\t"
7061      "L_aes_gcm_128_decrypt_ghash_aad_load_byte:\n\t"
7062        "lb         t0, (%[aad])\n\t"
7063        "sb         t0, (%[scratch])\n\t"
7064        "addi       %[aad], %[aad], 1\n\t"
7065        "addi       %[scratch], %[scratch], 1\n\t"
7066        "addi       t2, t2, -1\n\t"
7067        "bnez       t2, L_aes_gcm_128_decrypt_ghash_aad_load_byte\n\t"
7068        "sub        %[scratch], %[scratch], t1\n\t"
7069        "mv         t0, %[scratch]\n\t"
7070        VL1RE32_V(REG_V17, REG_T0)
7071        VGHSH_VV(REG_V18, REG_V17, REG_V19)
7072      "L_aes_gcm_128_decrypt_ghash_aad_done:\n\t"
7073        /* Done Hash in AAD */
7074
7075#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
7076        "mv         t0, %[rev_idx]\n\t"
7077        VL1RE32_V(REG_V15, REG_T0)
7078#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
7079        /* Load the counter. */
7080        "mv         t0, %[ctr]\n\t"
7081        VL1RE32_V(REG_V16, REG_T0)
7082#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
7083        VSLIDEDOWN_VI(REG_V20, REG_V16, 3)
7084#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
7085        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
7086        VRGATHER_VV(REG_V21, REG_V15, REG_V20)
7087        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
7088        VMV_V_V(REG_V20, REG_V21)
7089#else
7090        VREV8(REG_V20, REG_V20)
7091#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
7092#else
7093        "lw         t3, 12(%[ctr])\n\t"
7094        "slli       t3, t3, 32\n\t"
7095        REV8(REG_T3, REG_T3)
7096#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
7097
7098        /* Load key[0..7]. */
7099        "mv         t0, %[key]\n\t"
7100        VL8RE32_V(REG_V0, REG_T0)
7101        /* Load key[8..9]. */
7102        "addi       t0, t0, 128\n\t"
7103        VL2RE32_V(REG_V8, REG_T0)
7104        /* Load last round's key */
7105        "addi       t0, %[key], 224\n\t"
7106        VL1RE32_V(REG_V10, REG_T0)
7107
7108        "beqz       %[sz], L_aes_gcm_128_decrypt_blocks_done\n\t"
7109        "srli       t4, %[sz], 6\n\t"
7110        "beqz       t4, L_aes_gcm_128_decrypt_x4_blocks_done\n\t"
7111
7112        /* Calculate H^[1-4] - GMULT partials */
7113        VMV_V_V(REG_V21, REG_V19)
7114        VMV_V_V(REG_V22, REG_V19)
7115        /* Multiply H * H => H^2 */
7116        VGMUL_VV(REG_V21, REG_V19)
7117        VMV_V_V(REG_V23, REG_V21)
7118        /* Multiply H * H => H^3 */
7119        VGMUL_VV(REG_V22, REG_V21)
7120        /* Multiply H^2 * H^2 => H^4 */
7121        VGMUL_VV(REG_V23, REG_V21)
7122
7123      "L_aes_gcm_128_decrypt_x4_block_loop:\n\t"
7124        /* Load input. */
7125        "mv        t0, %[in]\n\t"
7126        VL4RE32_V(REG_V28, REG_T0)
7127        VMVR_V(REG_V24, REG_V28, 4)
7128        VGMUL_VV(REG_V24, REG_V23)
7129        VGMUL_VV(REG_V25, REG_V22)
7130        VGMUL_VV(REG_V26, REG_V21)
7131        VGMUL_VV(REG_V27, REG_V19)
7132        VXOR_VV(REG_V18, REG_V18, REG_V24)
7133        VXOR_VV(REG_V18, REG_V18, REG_V25)
7134        VXOR_VV(REG_V18, REG_V18, REG_V26)
7135        VXOR_VV(REG_V18, REG_V18, REG_V27)
7136        /* Calculate next 4 counters (+1-4) */
7137#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
7138        VADD_VI(REG_V20, REG_V20, 1)
7139        VMV_V_V(REG_V24, REG_V16)
7140#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
7141        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
7142        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
7143        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
7144#else
7145        VREV8(REG_V17, REG_V20)
7146#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
7147        VSLIDEUP_VI(REG_V24, REG_V17, 3)
7148        VADD_VI(REG_V20, REG_V20, 1)
7149        VMV_V_V(REG_V25, REG_V16)
7150#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
7151        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
7152        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
7153        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
7154#else
7155        VREV8(REG_V17, REG_V20)
7156#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
7157        VSLIDEUP_VI(REG_V25, REG_V17, 3)
7158        VADD_VI(REG_V20, REG_V20, 1)
7159        VMV_V_V(REG_V26, REG_V16)
7160#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
7161        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
7162        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
7163        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
7164#else
7165        VREV8(REG_V17, REG_V20)
7166#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
7167        VSLIDEUP_VI(REG_V26, REG_V17, 3)
7168        VADD_VI(REG_V20, REG_V20, 1)
7169        VMV_V_V(REG_V27, REG_V16)
7170#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
7171        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
7172        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
7173        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
7174#else
7175        VREV8(REG_V17, REG_V20)
7176#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
7177        VSLIDEUP_VI(REG_V27, REG_V17, 3)
7178#else
7179        "addi       t0, t3, 1\n\t"
7180        VMV_V_V(REG_V24, REG_V16)
7181        "addi       t1, t3, 2\n\t"
7182        VMV_V_V(REG_V25, REG_V16)
7183        "slli       t0, t0, 32\n\t"
7184        VMV_V_V(REG_V26, REG_V16)
7185        "slli       t1, t1, 32\n\t"
7186        VMV_V_V(REG_V27, REG_V16)
7187        REV8(REG_T0, REG_T0)
7188        REV8(REG_T1, REG_T1)
7189        VMV_V_X(REG_V20, REG_T0)
7190        "addi       t0, t3, 3\n\t"
7191        VSLIDEUP_VI(REG_V24, REG_V20, 3)
7192        "addi       t3, t3, 4\n\t"
7193        VMV_V_X(REG_V20, REG_T1)
7194        "slli       t0, t0, 32\n\t"
7195        VSLIDEUP_VI(REG_V25, REG_V20, 3)
7196        "slli       t1, t3, 32\n\t"
7197        REV8(REG_T0, REG_T0)
7198        REV8(REG_T1, REG_T1)
7199        VMV_V_X(REG_V20, REG_T0)
7200        VSLIDEUP_VI(REG_V26, REG_V20, 3)
7201        VMV_V_X(REG_V20, REG_T1)
7202        VSLIDEUP_VI(REG_V27, REG_V20, 3)
7203#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
7204
7205        VAESZ_VS(REG_V24, REG_V0)
7206        VAESZ_VS(REG_V25, REG_V0)
7207        VAESZ_VS(REG_V26, REG_V0)
7208        VAESZ_VS(REG_V27, REG_V0)
7209        VAESEM_VS(REG_V24, REG_V1)
7210        VAESEM_VS(REG_V24, REG_V2)
7211        VAESEM_VS(REG_V24, REG_V3)
7212        VAESEM_VS(REG_V24, REG_V4)
7213        VAESEM_VS(REG_V24, REG_V5)
7214        VAESEM_VS(REG_V24, REG_V6)
7215        VAESEM_VS(REG_V24, REG_V7)
7216        VAESEM_VS(REG_V24, REG_V8)
7217        VAESEM_VS(REG_V24, REG_V9)
7218        VAESEM_VS(REG_V25, REG_V1)
7219        VAESEM_VS(REG_V25, REG_V2)
7220        VAESEM_VS(REG_V25, REG_V3)
7221        VAESEM_VS(REG_V25, REG_V4)
7222        VAESEM_VS(REG_V25, REG_V5)
7223        VAESEM_VS(REG_V25, REG_V6)
7224        VAESEM_VS(REG_V25, REG_V7)
7225        VAESEM_VS(REG_V25, REG_V8)
7226        VAESEM_VS(REG_V25, REG_V9)
7227        VAESEM_VS(REG_V26, REG_V1)
7228        VAESEM_VS(REG_V26, REG_V2)
7229        VAESEM_VS(REG_V26, REG_V3)
7230        VAESEM_VS(REG_V26, REG_V4)
7231        VAESEM_VS(REG_V26, REG_V5)
7232        VAESEM_VS(REG_V26, REG_V6)
7233        VAESEM_VS(REG_V26, REG_V7)
7234        VAESEM_VS(REG_V26, REG_V8)
7235        VAESEM_VS(REG_V26, REG_V9)
7236        VAESEM_VS(REG_V27, REG_V1)
7237        VAESEM_VS(REG_V27, REG_V2)
7238        VAESEM_VS(REG_V27, REG_V3)
7239        VAESEM_VS(REG_V27, REG_V4)
7240        VAESEM_VS(REG_V27, REG_V5)
7241        VAESEM_VS(REG_V27, REG_V6)
7242        VAESEM_VS(REG_V27, REG_V7)
7243        VAESEM_VS(REG_V27, REG_V8)
7244        VAESEM_VS(REG_V27, REG_V9)
7245        VAESEF_VS(REG_V24, REG_V10)
7246        VAESEF_VS(REG_V25, REG_V10)
7247        VAESEF_VS(REG_V26, REG_V10)
7248        VAESEF_VS(REG_V27, REG_V10)
7249        VXOR_VV(REG_V28, REG_V24, REG_V28)
7250        VXOR_VV(REG_V29, REG_V25, REG_V29)
7251        VXOR_VV(REG_V30, REG_V26, REG_V30)
7252        VXOR_VV(REG_V31, REG_V27, REG_V31)
7253        /* Store output. */
7254        "mv         t0, %[out]\n\t"
7255        VS4R_V(REG_V28, REG_T0)
7256        "addi        %[in], %[in], 64\n\t"
7257        "addi        %[out], %[out], 64\n\t"
7258        /* Loop if more elements to process. */
7259        "addi       t4, t4, -1\n\t"
7260        "bnez       t4, L_aes_gcm_128_decrypt_x4_block_loop\n\t"
7261        "andi       %[sz], %[sz], 0x3f\n\t"
7262
7263      "L_aes_gcm_128_decrypt_x4_blocks_done:\n\t"
7264        "srli       t2, %[sz], 4\n\t"
7265        "beqz       t2, L_aes_gcm_128_decrypt_blocks_done\n\t"
7266
7267      "L_aes_gcm_128_decrypt_block_loop:\n\t"
7268#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
7269        VADD_VI(REG_V20, REG_V20, 1)
7270#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
7271        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
7272        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
7273        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
7274#else
7275        VREV8(REG_V17, REG_V20)
7276#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
7277        VMV_V_V(REG_V27, REG_V16)
7278        VSLIDEUP_VI(REG_V27, REG_V17, 3)
7279#else
7280        "addi       t3, t3, 1\n\t"
7281        "slli       t0, t3, 32\n\t"
7282        REV8(REG_T0, REG_T0)
7283        VMV_V_X(REG_V17, REG_T0)
7284        VMV_V_V(REG_V27, REG_V16)
7285        VSLIDEUP_VI(REG_V27, REG_V17, 3)
7286#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
7287
7288        VAESZ_VS(REG_V27, REG_V0)
7289        VAESEM_VS(REG_V27, REG_V1)
7290        VAESEM_VS(REG_V27, REG_V2)
7291        VAESEM_VS(REG_V27, REG_V3)
7292        VAESEM_VS(REG_V27, REG_V4)
7293        VAESEM_VS(REG_V27, REG_V5)
7294        VAESEM_VS(REG_V27, REG_V6)
7295        VAESEM_VS(REG_V27, REG_V7)
7296        VAESEM_VS(REG_V27, REG_V8)
7297        VAESEM_VS(REG_V27, REG_V9)
7298        VAESEF_VS(REG_V27, REG_V10)
7299
7300        /* Load input. */
7301        "mv         t0, %[in]\n\t"
7302        VL1RE32_V(REG_V17, REG_T0)
7303        VGHSH_VV(REG_V18, REG_V17, REG_V19)
7304        VXOR_VV(REG_V27, REG_V27, REG_V17)
7305        /* Store output. */
7306        "mv         t0, %[out]\n\t"
7307        VS1R_V(REG_V27, REG_T0)
7308
7309        "addi        %[in], %[in], 16\n\t"
7310        "addi        %[out], %[out], 16\n\t"
7311        /* Loop if more elements to process. */
7312        "addi       t2, t2, -1\n\t"
7313        "bnez       t2, L_aes_gcm_128_decrypt_block_loop\n\t"
7314
7315      "L_aes_gcm_128_decrypt_blocks_done:\n\t"
7316        "andi       t2, %[sz], 0xf\n\t"
7317        "beqz       t2, L_aes_gcm_128_decrypt_done\n\t"
7318
7319        VXOR_VV(REG_V17, REG_V17, REG_V17)
7320        "mv         t0, %[scratch]\n\t"
7321        VS1R_V(REG_V17, REG_T0)
7322        "mv         t1, t2\n\t"
7323      "L_aes_gcm_128_decrypt_load_byte:\n\t"
7324        "lb         t0, (%[in])\n\t"
7325        "sb         t0, (%[scratch])\n\t"
7326        "addi       %[in], %[in], 1\n\t"
7327        "addi       %[scratch], %[scratch], 1\n\t"
7328        "addi       t1, t1, -1\n\t"
7329        "bnez       t1, L_aes_gcm_128_decrypt_load_byte\n\t"
7330        "sub        %[scratch], %[scratch], t2\n\t"
7331        "mv         t0, %[scratch]\n\t"
7332        VL1RE32_V(REG_V17, REG_T0)
7333        VGHSH_VV(REG_V18, REG_V17, REG_V19)
7334
7335        /* Encrypt counter for partial block. */
7336#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
7337        VADD_VI(REG_V20, REG_V20, 1)
7338#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
7339        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
7340        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
7341        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
7342#else
7343        VREV8(REG_V17, REG_V20)
7344#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
7345        VMV_V_V(REG_V27, REG_V16)
7346        VSLIDEUP_VI(REG_V27, REG_V17, 3)
7347#else
7348        "addi       t3, t3, 1\n\t"
7349        "slli       t0, t3, 32\n\t"
7350        REV8(REG_T0, REG_T0)
7351        VMV_V_X(REG_V17, REG_T0)
7352        VMV_V_V(REG_V27, REG_V16)
7353        VSLIDEUP_VI(REG_V27, REG_V17, 3)
7354#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
7355
7356        VAESZ_VS(REG_V27, REG_V0)
7357        VAESEM_VS(REG_V27, REG_V1)
7358        VAESEM_VS(REG_V27, REG_V2)
7359        VAESEM_VS(REG_V27, REG_V3)
7360        VAESEM_VS(REG_V27, REG_V4)
7361        VAESEM_VS(REG_V27, REG_V5)
7362        VAESEM_VS(REG_V27, REG_V6)
7363        VAESEM_VS(REG_V27, REG_V7)
7364        VAESEM_VS(REG_V27, REG_V8)
7365        VAESEM_VS(REG_V27, REG_V9)
7366        VAESEF_VS(REG_V27, REG_V10)
7367
7368        /* Load scratch. */
7369        "mv         t0, %[scratch]\n\t"
7370        VL1RE32_V(REG_V17, REG_T0)
7371        VXOR_VV(REG_V27, REG_V27, REG_V17)
7372        /* Store scratch. */
7373        VS1R_V(REG_V27, REG_T0)
7374        "mv         t1, t2\n\t"
7375      "L_aes_gcm_128_decrypt_store_byte:\n\t"
7376        "lb         t0, (%[scratch])\n\t"
7377        "sb         t0, (%[out])\n\t"
7378        "addi       %[scratch], %[scratch], 1\n\t"
7379        "addi       %[out], %[out], 1\n\t"
7380        "addi       t1, t1, -1\n\t"
7381        "bnez       t1, L_aes_gcm_128_decrypt_store_byte\n\t"
7382        "sub        %[scratch], %[scratch], t2\n\t"
7383
7384      "L_aes_gcm_128_decrypt_done:\n\t"
7385
7386        /* Hash in the lengths of A and C in bits */
7387#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
7388        /* aSz is only 32-bits */
7389        /* Multiply by 8 do get size in bits. */
7390        "slli       %[aSz], %[aSz], 3\n\t"
7391        "srli       t0, %[aSz], 32\n\t"
7392        "srli       t1, %[aSz], 24\n\t"
7393        "srli       t2, %[aSz], 16\n\t"
7394        "srli       t3, %[aSz], 8\n\t"
7395        /* Top 3 bytes are 0. */
7396        "sh         x0    , 0(%[scratch])\n\t"
7397        "sb         x0    , 2(%[scratch])\n\t"
7398        "sb         t0    , 3(%[scratch])\n\t"
7399        "sb         t1    , 4(%[scratch])\n\t"
7400        "sb         t2    , 5(%[scratch])\n\t"
7401        "sb         t3    , 6(%[scratch])\n\t"
7402        "sb         %[aSz], 7(%[scratch])\n\t"
7403        /* sz is only 32-bits */
7404        /* Multiply by 8 do get size in bits. */
7405        "slli       %[sz], %[sz], 3\n\t"
7406        "srli       t0, %[sz], 32\n\t"
7407        "srli       t1, %[sz], 24\n\t"
7408        "srli       t2, %[sz], 16\n\t"
7409        "srli       t3, %[sz], 8\n\t"
7410        /* Top 3 bytes are 0. */
7411        "sh         x0   ,  8(%[scratch])\n\t"
7412        "sb         x0   , 10(%[scratch])\n\t"
7413        "sb         t0   , 11(%[scratch])\n\t"
7414        "sb         t1   , 12(%[scratch])\n\t"
7415        "sb         t2   , 13(%[scratch])\n\t"
7416        "sb         t3   , 14(%[scratch])\n\t"
7417        "sb         %[sz], 15(%[scratch])\n\t"
7418#else
7419        "slli       t0, %[aSz], 3\n\t"
7420        REV8(REG_T0, REG_T0)
7421        "sd         t0, 0(%[scratch])\n\t"
7422        "slli       t0, %[sz], 3\n\t"
7423        REV8(REG_T0, REG_T0)
7424        "sd         t0, 8(%[scratch])\n\t"
7425#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
7426        "mv         t0, %[scratch]\n\t"
7427        VL1RE32_V(REG_V17, REG_T0)
7428        VGHSH_VV(REG_V18, REG_V17, REG_V19)
7429
7430        VAESZ_VS(REG_V16, REG_V0)
7431        VAESEM_VS(REG_V16, REG_V1)
7432        VAESEM_VS(REG_V16, REG_V2)
7433        VAESEM_VS(REG_V16, REG_V3)
7434        VAESEM_VS(REG_V16, REG_V4)
7435        VAESEM_VS(REG_V16, REG_V5)
7436        VAESEM_VS(REG_V16, REG_V6)
7437        VAESEM_VS(REG_V16, REG_V7)
7438        VAESEM_VS(REG_V16, REG_V8)
7439        VAESEM_VS(REG_V16, REG_V9)
7440        VAESEF_VS(REG_V16, REG_V10)
7441        VXOR_VV(REG_V18, REG_V18, REG_V16)
7442
7443        "li         t1, 16\n\t"
7444        "blt        %[tagSz], t1, L_aes_gcm_128_decrypt_tag_small\n\t"
7445        "mv         t0, %[tag]\n\t"
7446        VL1RE32_V(REG_V17, REG_T0)
7447        VXOR_VV(REG_V19, REG_V19, REG_V19)
7448        VXOR_VV(REG_V18, REG_V18, REG_V17)
7449        VMSNE_VV(REG_V19, REG_V19, REG_V18)
7450        VCPOP_M(REG_T0, REG_V19)
7451        "beqz       x0, L_aes_gcm_128_decrypt_tag_done\n\t"
7452      "L_aes_gcm_128_decrypt_tag_small:\n\t"
7453        "mv         t0, %[scratch]\n\t"
7454        VS1R_V(REG_V18, REG_T0)
7455        "mv         t1, %[tagSz]\n\t"
7456        "xor        t0, t0, t0\n\t"
7457      "L_aes_gcm_128_decrypt_store_tag_byte:\n\t"
7458        "lb         t2, (%[scratch])\n\t"
7459        "lb         t3, (%[tag])\n\t"
7460        "xor        t3, t3, t2\n\t"
7461        "or         t0, t0, t3\n\t"
7462        "addi       %[scratch], %[scratch], 1\n\t"
7463        "addi       %[tag], %[tag], 1\n\t"
7464        "addi       t1, t1, -1\n\t"
7465        "bnez       t1, L_aes_gcm_128_decrypt_store_tag_byte\n\t"
7466      "L_aes_gcm_128_decrypt_tag_done:\n\t"
7467        "negw       t0, t0\n\t"
7468        "sraiw      t0, t0, 31\n\t"
7469        "andi       %[ret], t0, -180\n\t"
7470
7471        : [out] "+r" (out), [in] "+r" (in), [key] "+r" (key),
7472          [aSz] "+r" (aadSz), [aad] "+r" (aad), [ret] "+r" (ret),
7473          [sz] "+r" (sz)
7474        : [ctr] "r" (ctr), [scratch] "r" (scratch),
7475          [h] "r" (aes->gcm.H), [tag] "r" (tag), [tagSz] "r" (tagSz)
7476#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
7477          , [rev_idx] "r" (rev_idx)
7478#endif
7479        : "memory", "t0", "t1", "t2", "t3", "t4"
7480    );
7481
7482#ifdef OPENSSL_EXTRA
7483    if ((tag != NULL) && (in != NULL) && (sz != 0)) {
7484        /* store AAD size for next call */
7485        aes->gcm.aadLen = aadSz;
7486    }
7487#endif
7488
7489    return ret;
7490}
7491#endif /* WOLFSSL_AES_128 */
7492
7493#ifdef WOLFSSL_AES_192
7494/* Decrypt data using AES-192-GCM.
7495 *
7496 * @param [in]  aes      AES object.
7497 * @param [out] out      Decrypted data.
7498 * @param [in]  in       Data to decrypt and GHASH.
7499 * @param [in]  sz       Number of bytes of data.
7500 * @param [in]  nonce    Nonce used to calculate first IV.
7501 * @param [in]  nonceSz  Length of nonce in bytes.
7502 * @param [out] tag      Authentication tag.
7503 * @param [in]  tagSz    Length of authentication tag in bytes.
7504 * @param [in]  aad      Additional Authentication Data (AAD).
7505 * @param [in]  aadSz    Length of AAD in bytes.
7506 * @return  0 on success.
7507 * @return  AES_GCM_AUTH_E when authentication tag computed doesn't match
7508 *          tag passed in.
7509 */
7510static int Aes192GcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
7511    const byte* nonce, word32 nonceSz, const byte* tag, word32 tagSz,
7512    const byte* aad, word32 aadSz)
7513{
7514    int ret = 0;
7515    byte counter[WC_AES_BLOCK_SIZE];
7516    byte scratch[WC_AES_BLOCK_SIZE];
7517    /* Noticed different optimization levels treated head of array different.
7518     * Some cases was stack pointer plus offset others was a register containing
7519     * address. To make uniform for passing in to inline assembly code am using
7520     * pointers to the head of each local array.
7521     */
7522    byte* ctr  = counter;
7523    byte* key = (byte*)aes->key;
7524
7525    XMEMSET(counter, 0, WC_AES_BLOCK_SIZE);
7526    if (nonceSz == GCM_NONCE_MID_SZ) {
7527        XMEMCPY(counter, nonce, GCM_NONCE_MID_SZ);
7528        counter[WC_AES_BLOCK_SIZE - 1] = 1;
7529    }
7530    else {
7531#ifdef OPENSSL_EXTRA
7532        word32 aadTemp = aes->gcm.aadLen;
7533        aes->gcm.aadLen = 0;
7534#endif
7535        GHASH(&aes->gcm, NULL, 0, nonce, nonceSz, counter, WC_AES_BLOCK_SIZE);
7536#ifdef OPENSSL_EXTRA
7537        aes->gcm.aadLen = aadTemp;
7538#endif
7539    }
7540
7541    __asm__ __volatile__ (
7542        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
7543
7544        /* X=0, get H */
7545        VXOR_VV(REG_V18, REG_V18, REG_V18)
7546        "mv         t0, %[h]\n\t"
7547        VL1RE32_V(REG_V19, REG_T0)
7548
7549        /* Hash in AAD, the Additional Authentication Data */
7550        "beqz       %[aSz], L_aes_gcm_192_decrypt_ghash_aad_done\n\t"
7551        "beqz       %[aad], L_aes_gcm_192_decrypt_ghash_aad_done\n\t"
7552
7553        "srli       t1, %[aSz], 4\n\t"
7554        "beqz       t1, L_aes_gcm_192_decrypt_ghash_aad_blocks_done\n\t"
7555
7556      "L_aes_gcm_192_decrypt_ghash_aad_loop:\n\t"
7557        "mv         t0, %[aad]\n\t"
7558        VL1RE32_V(REG_V17, REG_T0)
7559        VGHSH_VV(REG_V18, REG_V17, REG_V19)
7560        "addi       %[aad], %[aad], 16\n\t"
7561        "addi       t1, t1, -1\n\t"
7562        "bnez       t1, L_aes_gcm_192_decrypt_ghash_aad_loop\n\t"
7563      "L_aes_gcm_192_decrypt_ghash_aad_blocks_done:\n\t"
7564        "andi       t1, %[aSz], 0xf\n\t"
7565        "beqz       t1, L_aes_gcm_192_decrypt_ghash_aad_done\n\t"
7566        VXOR_VV(REG_V17, REG_V17, REG_V17)
7567        "mv         t0, %[scratch]\n\t"
7568        VS1R_V(REG_V17, REG_T0)
7569        "mv         t2, t1\n\t"
7570      "L_aes_gcm_192_decrypt_ghash_aad_load_byte:\n\t"
7571        "lb         t0, (%[aad])\n\t"
7572        "sb         t0, (%[scratch])\n\t"
7573        "addi       %[aad], %[aad], 1\n\t"
7574        "addi       %[scratch], %[scratch], 1\n\t"
7575        "addi       t2, t2, -1\n\t"
7576        "bnez       t2, L_aes_gcm_192_decrypt_ghash_aad_load_byte\n\t"
7577        "sub        %[scratch], %[scratch], t1\n\t"
7578        "mv         t0, %[scratch]\n\t"
7579        VL1RE32_V(REG_V17, REG_T0)
7580        VGHSH_VV(REG_V18, REG_V17, REG_V19)
7581      "L_aes_gcm_192_decrypt_ghash_aad_done:\n\t"
7582        /* Done Hash in AAD */
7583
7584#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
7585        "mv         t0, %[rev_idx]\n\t"
7586        VL1RE32_V(REG_V15, REG_T0)
7587#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
7588        /* Load the counter. */
7589        "mv         t0, %[ctr]\n\t"
7590        VL1RE32_V(REG_V16, REG_T0)
7591#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
7592        VSLIDEDOWN_VI(REG_V20, REG_V16, 3)
7593#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
7594        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
7595        VRGATHER_VV(REG_V21, REG_V15, REG_V20)
7596        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
7597        VMV_V_V(REG_V20, REG_V21)
7598#else
7599        VREV8(REG_V20, REG_V20)
7600#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
7601#else
7602        "lw         t3, 12(%[ctr])\n\t"
7603        "slli       t3, t3, 32\n\t"
7604        REV8(REG_T3, REG_T3)
7605#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
7606
7607        /* Load key[0..7]. */
7608        "mv         t0, %[key]\n\t"
7609        VL8RE32_V(REG_V0, REG_T0)
7610        /* Load key[8..11]. */
7611        "addi       t0, t0, 128\n\t"
7612        VL4RE32_V(REG_V8, REG_T0)
7613        /* Load last round's key */
7614        "addi       t0, %[key], 224\n\t"
7615        VL1RE32_V(REG_V12, REG_T0)
7616
7617        "beqz       %[sz], L_aes_gcm_192_decrypt_blocks_done\n\t"
7618        "srli       t4, %[sz], 6\n\t"
7619        "beqz       t4, L_aes_gcm_192_decrypt_x4_blocks_done\n\t"
7620
7621        /* Calculate H^[1-4] - GMULT partials */
7622        VMV_V_V(REG_V21, REG_V19)
7623        VMV_V_V(REG_V22, REG_V19)
7624        /* Multiply H * H => H^2 */
7625        VGMUL_VV(REG_V21, REG_V19)
7626        VMV_V_V(REG_V23, REG_V21)
7627        /* Multiply H * H => H^3 */
7628        VGMUL_VV(REG_V22, REG_V21)
7629        /* Multiply H^2 * H^2 => H^4 */
7630        VGMUL_VV(REG_V23, REG_V21)
7631
7632      "L_aes_gcm_192_decrypt_x4_block_loop:\n\t"
7633        /* Load input. */
7634        "mv        t0, %[in]\n\t"
7635        VL4RE32_V(REG_V28, REG_T0)
7636        VMVR_V(REG_V24, REG_V28, 4)
7637        VGMUL_VV(REG_V24, REG_V23)
7638        VGMUL_VV(REG_V25, REG_V22)
7639        VGMUL_VV(REG_V26, REG_V21)
7640        VGMUL_VV(REG_V27, REG_V19)
7641        VXOR_VV(REG_V18, REG_V18, REG_V24)
7642        VXOR_VV(REG_V18, REG_V18, REG_V25)
7643        VXOR_VV(REG_V18, REG_V18, REG_V26)
7644        VXOR_VV(REG_V18, REG_V18, REG_V27)
7645        /* Calculate next 4 counters (+1-4) */
7646#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
7647        VADD_VI(REG_V20, REG_V20, 1)
7648        VMV_V_V(REG_V24, REG_V16)
7649#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
7650        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
7651        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
7652        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
7653#else
7654        VREV8(REG_V17, REG_V20)
7655#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
7656        VSLIDEUP_VI(REG_V24, REG_V17, 3)
7657        VADD_VI(REG_V20, REG_V20, 1)
7658        VMV_V_V(REG_V25, REG_V16)
7659#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
7660        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
7661        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
7662        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
7663#else
7664        VREV8(REG_V17, REG_V20)
7665#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
7666        VSLIDEUP_VI(REG_V25, REG_V17, 3)
7667        VADD_VI(REG_V20, REG_V20, 1)
7668        VMV_V_V(REG_V26, REG_V16)
7669#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
7670        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
7671        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
7672        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
7673#else
7674        VREV8(REG_V17, REG_V20)
7675#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
7676        VSLIDEUP_VI(REG_V26, REG_V17, 3)
7677        VADD_VI(REG_V20, REG_V20, 1)
7678        VMV_V_V(REG_V27, REG_V16)
7679#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
7680        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
7681        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
7682        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
7683#else
7684        VREV8(REG_V17, REG_V20)
7685#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
7686        VSLIDEUP_VI(REG_V27, REG_V17, 3)
7687#else
7688        "addi       t0, t3, 1\n\t"
7689        VMV_V_V(REG_V24, REG_V16)
7690        "addi       t1, t3, 2\n\t"
7691        VMV_V_V(REG_V25, REG_V16)
7692        "slli       t0, t0, 32\n\t"
7693        VMV_V_V(REG_V26, REG_V16)
7694        "slli       t1, t1, 32\n\t"
7695        VMV_V_V(REG_V27, REG_V16)
7696        REV8(REG_T0, REG_T0)
7697        REV8(REG_T1, REG_T1)
7698        VMV_V_X(REG_V20, REG_T0)
7699        "addi       t0, t3, 3\n\t"
7700        VSLIDEUP_VI(REG_V24, REG_V20, 3)
7701        "addi       t3, t3, 4\n\t"
7702        VMV_V_X(REG_V20, REG_T1)
7703        "slli       t0, t0, 32\n\t"
7704        VSLIDEUP_VI(REG_V25, REG_V20, 3)
7705        "slli       t1, t3, 32\n\t"
7706        REV8(REG_T0, REG_T0)
7707        REV8(REG_T1, REG_T1)
7708        VMV_V_X(REG_V20, REG_T0)
7709        VSLIDEUP_VI(REG_V26, REG_V20, 3)
7710        VMV_V_X(REG_V20, REG_T1)
7711        VSLIDEUP_VI(REG_V27, REG_V20, 3)
7712#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
7713
7714        VAESZ_VS(REG_V24, REG_V0)
7715        VAESZ_VS(REG_V25, REG_V0)
7716        VAESZ_VS(REG_V26, REG_V0)
7717        VAESZ_VS(REG_V27, REG_V0)
7718        VAESEM_VS(REG_V24, REG_V1)
7719        VAESEM_VS(REG_V24, REG_V2)
7720        VAESEM_VS(REG_V24, REG_V3)
7721        VAESEM_VS(REG_V24, REG_V4)
7722        VAESEM_VS(REG_V24, REG_V5)
7723        VAESEM_VS(REG_V24, REG_V6)
7724        VAESEM_VS(REG_V24, REG_V7)
7725        VAESEM_VS(REG_V24, REG_V8)
7726        VAESEM_VS(REG_V24, REG_V9)
7727        VAESEM_VS(REG_V24, REG_V10)
7728        VAESEM_VS(REG_V24, REG_V11)
7729        VAESEM_VS(REG_V25, REG_V1)
7730        VAESEM_VS(REG_V25, REG_V2)
7731        VAESEM_VS(REG_V25, REG_V3)
7732        VAESEM_VS(REG_V25, REG_V4)
7733        VAESEM_VS(REG_V25, REG_V5)
7734        VAESEM_VS(REG_V25, REG_V6)
7735        VAESEM_VS(REG_V25, REG_V7)
7736        VAESEM_VS(REG_V25, REG_V8)
7737        VAESEM_VS(REG_V25, REG_V9)
7738        VAESEM_VS(REG_V25, REG_V10)
7739        VAESEM_VS(REG_V25, REG_V11)
7740        VAESEM_VS(REG_V26, REG_V1)
7741        VAESEM_VS(REG_V26, REG_V2)
7742        VAESEM_VS(REG_V26, REG_V3)
7743        VAESEM_VS(REG_V26, REG_V4)
7744        VAESEM_VS(REG_V26, REG_V5)
7745        VAESEM_VS(REG_V26, REG_V6)
7746        VAESEM_VS(REG_V26, REG_V7)
7747        VAESEM_VS(REG_V26, REG_V8)
7748        VAESEM_VS(REG_V26, REG_V9)
7749        VAESEM_VS(REG_V26, REG_V10)
7750        VAESEM_VS(REG_V26, REG_V11)
7751        VAESEM_VS(REG_V27, REG_V1)
7752        VAESEM_VS(REG_V27, REG_V2)
7753        VAESEM_VS(REG_V27, REG_V3)
7754        VAESEM_VS(REG_V27, REG_V4)
7755        VAESEM_VS(REG_V27, REG_V5)
7756        VAESEM_VS(REG_V27, REG_V6)
7757        VAESEM_VS(REG_V27, REG_V7)
7758        VAESEM_VS(REG_V27, REG_V8)
7759        VAESEM_VS(REG_V27, REG_V9)
7760        VAESEM_VS(REG_V27, REG_V10)
7761        VAESEM_VS(REG_V27, REG_V11)
7762        VAESEF_VS(REG_V24, REG_V12)
7763        VAESEF_VS(REG_V25, REG_V12)
7764        VAESEF_VS(REG_V26, REG_V12)
7765        VAESEF_VS(REG_V27, REG_V12)
7766        VXOR_VV(REG_V28, REG_V24, REG_V28)
7767        VXOR_VV(REG_V29, REG_V25, REG_V29)
7768        VXOR_VV(REG_V30, REG_V26, REG_V30)
7769        VXOR_VV(REG_V31, REG_V27, REG_V31)
7770        /* Store output. */
7771        "mv         t0, %[out]\n\t"
7772        VS4R_V(REG_V28, REG_T0)
7773        "addi        %[in], %[in], 64\n\t"
7774        "addi        %[out], %[out], 64\n\t"
7775        /* Loop if more elements to process. */
7776        "addi       t4, t4, -1\n\t"
7777        "bnez       t4, L_aes_gcm_192_decrypt_x4_block_loop\n\t"
7778        "andi       %[sz], %[sz], 0x3f\n\t"
7779
7780      "L_aes_gcm_192_decrypt_x4_blocks_done:\n\t"
7781        "srli       t2, %[sz], 4\n\t"
7782        "beqz       t2, L_aes_gcm_192_decrypt_blocks_done\n\t"
7783
7784      "L_aes_gcm_192_decrypt_block_loop:\n\t"
7785#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
7786        VADD_VI(REG_V20, REG_V20, 1)
7787#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
7788        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
7789        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
7790        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
7791#else
7792        VREV8(REG_V17, REG_V20)
7793#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
7794        VMV_V_V(REG_V27, REG_V16)
7795        VSLIDEUP_VI(REG_V27, REG_V17, 3)
7796#else
7797        "addi       t3, t3, 1\n\t"
7798        "slli       t0, t3, 32\n\t"
7799        REV8(REG_T0, REG_T0)
7800        VMV_V_X(REG_V17, REG_T0)
7801        VMV_V_V(REG_V27, REG_V16)
7802        VSLIDEUP_VI(REG_V27, REG_V17, 3)
7803#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
7804
7805        VAESZ_VS(REG_V27, REG_V0)
7806        VAESEM_VS(REG_V27, REG_V1)
7807        VAESEM_VS(REG_V27, REG_V2)
7808        VAESEM_VS(REG_V27, REG_V3)
7809        VAESEM_VS(REG_V27, REG_V4)
7810        VAESEM_VS(REG_V27, REG_V5)
7811        VAESEM_VS(REG_V27, REG_V6)
7812        VAESEM_VS(REG_V27, REG_V7)
7813        VAESEM_VS(REG_V27, REG_V8)
7814        VAESEM_VS(REG_V27, REG_V9)
7815        VAESEM_VS(REG_V27, REG_V10)
7816        VAESEM_VS(REG_V27, REG_V11)
7817        VAESEF_VS(REG_V27, REG_V12)
7818
7819        /* Load input. */
7820        "mv         t0, %[in]\n\t"
7821        VL1RE32_V(REG_V17, REG_T0)
7822        VGHSH_VV(REG_V18, REG_V17, REG_V19)
7823        VXOR_VV(REG_V27, REG_V27, REG_V17)
7824        /* Store output. */
7825        "mv         t0, %[out]\n\t"
7826        VS1R_V(REG_V27, REG_T0)
7827
7828        "addi        %[in], %[in], 16\n\t"
7829        "addi        %[out], %[out], 16\n\t"
7830        /* Loop if more elements to process. */
7831        "addi       t2, t2, -1\n\t"
7832        "bnez       t2, L_aes_gcm_192_decrypt_block_loop\n\t"
7833
7834      "L_aes_gcm_192_decrypt_blocks_done:\n\t"
7835        "andi       t2, %[sz], 0xf\n\t"
7836        "beqz       t2, L_aes_gcm_192_decrypt_done\n\t"
7837
7838        VXOR_VV(REG_V17, REG_V17, REG_V17)
7839        "mv         t0, %[scratch]\n\t"
7840        VS1R_V(REG_V17, REG_T0)
7841        "mv         t1, t2\n\t"
7842      "L_aes_gcm_192_decrypt_load_byte:\n\t"
7843        "lb         t0, (%[in])\n\t"
7844        "sb         t0, (%[scratch])\n\t"
7845        "addi       %[in], %[in], 1\n\t"
7846        "addi       %[scratch], %[scratch], 1\n\t"
7847        "addi       t1, t1, -1\n\t"
7848        "bnez       t1, L_aes_gcm_192_decrypt_load_byte\n\t"
7849        "sub        %[scratch], %[scratch], t2\n\t"
7850        "mv         t0, %[scratch]\n\t"
7851        VL1RE32_V(REG_V17, REG_T0)
7852        VGHSH_VV(REG_V18, REG_V17, REG_V19)
7853
7854        /* Encrypt counter for partial block. */
7855#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
7856        VADD_VI(REG_V20, REG_V20, 1)
7857#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
7858        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
7859        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
7860        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
7861#else
7862        VREV8(REG_V17, REG_V20)
7863#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
7864        VMV_V_V(REG_V27, REG_V16)
7865        VSLIDEUP_VI(REG_V27, REG_V17, 3)
7866#else
7867        "addi       t3, t3, 1\n\t"
7868        "slli       t0, t3, 32\n\t"
7869        REV8(REG_T0, REG_T0)
7870        VMV_V_X(REG_V17, REG_T0)
7871        VMV_V_V(REG_V27, REG_V16)
7872        VSLIDEUP_VI(REG_V27, REG_V17, 3)
7873#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
7874
7875        VAESZ_VS(REG_V27, REG_V0)
7876        VAESEM_VS(REG_V27, REG_V1)
7877        VAESEM_VS(REG_V27, REG_V2)
7878        VAESEM_VS(REG_V27, REG_V3)
7879        VAESEM_VS(REG_V27, REG_V4)
7880        VAESEM_VS(REG_V27, REG_V5)
7881        VAESEM_VS(REG_V27, REG_V6)
7882        VAESEM_VS(REG_V27, REG_V7)
7883        VAESEM_VS(REG_V27, REG_V8)
7884        VAESEM_VS(REG_V27, REG_V9)
7885        VAESEM_VS(REG_V27, REG_V10)
7886        VAESEM_VS(REG_V27, REG_V11)
7887        VAESEF_VS(REG_V27, REG_V12)
7888
7889        /* Load scratch. */
7890        "mv         t0, %[scratch]\n\t"
7891        VL1RE32_V(REG_V17, REG_T0)
7892        VXOR_VV(REG_V27, REG_V27, REG_V17)
7893        /* Store scratch. */
7894        VS1R_V(REG_V27, REG_T0)
7895        "mv         t1, t2\n\t"
7896      "L_aes_gcm_192_decrypt_store_byte:\n\t"
7897        "lb         t0, (%[scratch])\n\t"
7898        "sb         t0, (%[out])\n\t"
7899        "addi       %[scratch], %[scratch], 1\n\t"
7900        "addi       %[out], %[out], 1\n\t"
7901        "addi       t1, t1, -1\n\t"
7902        "bnez       t1, L_aes_gcm_192_decrypt_store_byte\n\t"
7903        "sub        %[scratch], %[scratch], t2\n\t"
7904
7905      "L_aes_gcm_192_decrypt_done:\n\t"
7906
7907        /* Hash in the lengths of A and C in bits */
7908#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
7909        /* aSz is only 32-bits */
7910        /* Multiply by 8 do get size in bits. */
7911        "slli       %[aSz], %[aSz], 3\n\t"
7912        "srli       t0, %[aSz], 32\n\t"
7913        "srli       t1, %[aSz], 24\n\t"
7914        "srli       t2, %[aSz], 16\n\t"
7915        "srli       t3, %[aSz], 8\n\t"
7916        /* Top 3 bytes are 0. */
7917        "sh         x0    , 0(%[scratch])\n\t"
7918        "sb         x0    , 2(%[scratch])\n\t"
7919        "sb         t0    , 3(%[scratch])\n\t"
7920        "sb         t1    , 4(%[scratch])\n\t"
7921        "sb         t2    , 5(%[scratch])\n\t"
7922        "sb         t3    , 6(%[scratch])\n\t"
7923        "sb         %[aSz], 7(%[scratch])\n\t"
7924        /* sz is only 32-bits */
7925        /* Multiply by 8 do get size in bits. */
7926        "slli       %[sz], %[sz], 3\n\t"
7927        "srli       t0, %[sz], 32\n\t"
7928        "srli       t1, %[sz], 24\n\t"
7929        "srli       t2, %[sz], 16\n\t"
7930        "srli       t3, %[sz], 8\n\t"
7931        /* Top 3 bytes are 0. */
7932        "sh         x0   ,  8(%[scratch])\n\t"
7933        "sb         x0   , 10(%[scratch])\n\t"
7934        "sb         t0   , 11(%[scratch])\n\t"
7935        "sb         t1   , 12(%[scratch])\n\t"
7936        "sb         t2   , 13(%[scratch])\n\t"
7937        "sb         t3   , 14(%[scratch])\n\t"
7938        "sb         %[sz], 15(%[scratch])\n\t"
7939#else
7940        "slli       t0, %[aSz], 3\n\t"
7941        REV8(REG_T0, REG_T0)
7942        "sd         t0, 0(%[scratch])\n\t"
7943        "slli       t0, %[sz], 3\n\t"
7944        REV8(REG_T0, REG_T0)
7945        "sd         t0, 8(%[scratch])\n\t"
7946#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
7947        "mv         t0, %[scratch]\n\t"
7948        VL1RE32_V(REG_V17, REG_T0)
7949        VGHSH_VV(REG_V18, REG_V17, REG_V19)
7950
7951        VAESZ_VS(REG_V16, REG_V0)
7952        VAESEM_VS(REG_V16, REG_V1)
7953        VAESEM_VS(REG_V16, REG_V2)
7954        VAESEM_VS(REG_V16, REG_V3)
7955        VAESEM_VS(REG_V16, REG_V4)
7956        VAESEM_VS(REG_V16, REG_V5)
7957        VAESEM_VS(REG_V16, REG_V6)
7958        VAESEM_VS(REG_V16, REG_V7)
7959        VAESEM_VS(REG_V16, REG_V8)
7960        VAESEM_VS(REG_V16, REG_V9)
7961        VAESEM_VS(REG_V16, REG_V10)
7962        VAESEM_VS(REG_V16, REG_V11)
7963        VAESEF_VS(REG_V16, REG_V12)
7964        VXOR_VV(REG_V18, REG_V18, REG_V16)
7965
7966        "li         t1, 16\n\t"
7967        "blt        %[tagSz], t1, L_aes_gcm_192_decrypt_tag_small\n\t"
7968        "mv         t0, %[tag]\n\t"
7969        VL1RE32_V(REG_V17, REG_T0)
7970        VXOR_VV(REG_V19, REG_V19, REG_V19)
7971        VXOR_VV(REG_V18, REG_V18, REG_V17)
7972        VMSNE_VV(REG_V19, REG_V19, REG_V18)
7973        VCPOP_M(REG_T0, REG_V19)
7974        "beqz       x0, L_aes_gcm_192_decrypt_tag_done\n\t"
7975      "L_aes_gcm_192_decrypt_tag_small:\n\t"
7976        "mv         t0, %[scratch]\n\t"
7977        VS1R_V(REG_V18, REG_T0)
7978        "mv         t1, %[tagSz]\n\t"
7979        "xor        t0, t0, t0\n\t"
7980      "L_aes_gcm_192_decrypt_store_tag_byte:\n\t"
7981        "lb         t2, (%[scratch])\n\t"
7982        "lb         t3, (%[tag])\n\t"
7983        "xor        t3, t3, t2\n\t"
7984        "or         t0, t0, t3\n\t"
7985        "addi       %[scratch], %[scratch], 1\n\t"
7986        "addi       %[tag], %[tag], 1\n\t"
7987        "addi       t1, t1, -1\n\t"
7988        "bnez       t1, L_aes_gcm_192_decrypt_store_tag_byte\n\t"
7989      "L_aes_gcm_192_decrypt_tag_done:\n\t"
7990        "negw       t0, t0\n\t"
7991        "sraiw      t0, t0, 31\n\t"
7992        "andi       %[ret], t0, -180\n\t"
7993
7994        : [out] "+r" (out), [in] "+r" (in), [key] "+r" (key),
7995          [aSz] "+r" (aadSz), [aad] "+r" (aad), [ret] "+r" (ret),
7996          [sz] "+r" (sz)
7997        : [ctr] "r" (ctr), [scratch] "r" (scratch),
7998          [h] "r" (aes->gcm.H), [tag] "r" (tag), [tagSz] "r" (tagSz)
7999#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
8000          , [rev_idx] "r" (rev_idx)
8001#endif
8002        : "memory", "t0", "t1", "t2", "t3", "t4"
8003    );
8004
8005#ifdef OPENSSL_EXTRA
8006    if ((tag != NULL) && (in != NULL) && (sz != 0)) {
8007        /* store AAD size for next call */
8008        aes->gcm.aadLen = aadSz;
8009    }
8010#endif
8011
8012    return ret;
8013}
8014#endif /* WOLFSSL_AES_192 */
8015
8016#ifdef WOLFSSL_AES_256
8017/* Decrypt data using AES-256-GCM.
8018 *
8019 * @param [in]  aes      AES object.
8020 * @param [out] out      Decrypted data.
8021 * @param [in]  in       Data to decrypt and GHASH.
8022 * @param [in]  sz       Number of bytes of data.
8023 * @param [in]  nonce    Nonce used to calculate first IV.
8024 * @param [in]  nonceSz  Length of nonce in bytes.
8025 * @param [out] tag      Authentication tag.
8026 * @param [in]  tagSz    Length of authentication tag in bytes.
8027 * @param [in]  aad      Additional Authentication Data (AAD).
8028 * @param [in]  aadSz    Length of AAD in bytes.
8029 * @return  0 on success.
8030 * @return  AES_GCM_AUTH_E when authentication tag computed doesn't match
8031 *          tag passed in.
8032 */
8033static int Aes256GcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
8034    const byte* nonce, word32 nonceSz, const byte* tag, word32 tagSz,
8035    const byte* aad, word32 aadSz)
8036{
8037    int ret = 0;
8038    byte counter[WC_AES_BLOCK_SIZE];
8039    byte scratch[WC_AES_BLOCK_SIZE];
8040    /* Noticed different optimization levels treated head of array different.
8041     * Some cases was stack pointer plus offset others was a register containing
8042     * address. To make uniform for passing in to inline assembly code am using
8043     * pointers to the head of each local array.
8044     */
8045    byte* ctr  = counter;
8046    byte* key = (byte*)aes->key;
8047
8048    XMEMSET(counter, 0, WC_AES_BLOCK_SIZE);
8049    if (nonceSz == GCM_NONCE_MID_SZ) {
8050        XMEMCPY(counter, nonce, GCM_NONCE_MID_SZ);
8051        counter[WC_AES_BLOCK_SIZE - 1] = 1;
8052    }
8053    else {
8054#ifdef OPENSSL_EXTRA
8055        word32 aadTemp = aes->gcm.aadLen;
8056        aes->gcm.aadLen = 0;
8057#endif
8058        GHASH(&aes->gcm, NULL, 0, nonce, nonceSz, counter, WC_AES_BLOCK_SIZE);
8059#ifdef OPENSSL_EXTRA
8060        aes->gcm.aadLen = aadTemp;
8061#endif
8062    }
8063
8064    __asm__ __volatile__ (
8065        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
8066
8067        /* X=0, get H */
8068        VXOR_VV(REG_V18, REG_V18, REG_V18)
8069        "mv         t0, %[h]\n\t"
8070        VL1RE32_V(REG_V19, REG_T0)
8071
8072        /* Hash in AAD, the Additional Authentication Data */
8073        "beqz       %[aSz], L_aes_gcm_256_decrypt_ghash_aad_done\n\t"
8074        "beqz       %[aad], L_aes_gcm_256_decrypt_ghash_aad_done\n\t"
8075
8076        "srli       t1, %[aSz], 4\n\t"
8077        "beqz       t1, L_aes_gcm_256_decrypt_ghash_aad_blocks_done\n\t"
8078
8079      "L_aes_gcm_256_decrypt_ghash_aad_loop:\n\t"
8080        "mv         t0, %[aad]\n\t"
8081        VL1RE32_V(REG_V17, REG_T0)
8082        VGHSH_VV(REG_V18, REG_V17, REG_V19)
8083        "addi       %[aad], %[aad], 16\n\t"
8084        "addi       t1, t1, -1\n\t"
8085        "bnez       t1, L_aes_gcm_256_decrypt_ghash_aad_loop\n\t"
8086      "L_aes_gcm_256_decrypt_ghash_aad_blocks_done:\n\t"
8087        "andi       t1, %[aSz], 0xf\n\t"
8088        "beqz       t1, L_aes_gcm_256_decrypt_ghash_aad_done\n\t"
8089        VXOR_VV(REG_V17, REG_V17, REG_V17)
8090        "mv         t0, %[scratch]\n\t"
8091        VS1R_V(REG_V17, REG_T0)
8092        "mv         t2, t1\n\t"
8093      "L_aes_gcm_256_decrypt_ghash_aad_load_byte:\n\t"
8094        "lb         t0, (%[aad])\n\t"
8095        "sb         t0, (%[scratch])\n\t"
8096        "addi       %[aad], %[aad], 1\n\t"
8097        "addi       %[scratch], %[scratch], 1\n\t"
8098        "addi       t2, t2, -1\n\t"
8099        "bnez       t2, L_aes_gcm_256_decrypt_ghash_aad_load_byte\n\t"
8100        "sub        %[scratch], %[scratch], t1\n\t"
8101        "mv         t0, %[scratch]\n\t"
8102        VL1RE32_V(REG_V17, REG_T0)
8103        VGHSH_VV(REG_V18, REG_V17, REG_V19)
8104      "L_aes_gcm_256_decrypt_ghash_aad_done:\n\t"
8105        /* Done Hash in AAD */
8106
8107#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
8108        "mv         t0, %[rev_idx]\n\t"
8109        VL1RE32_V(REG_V15, REG_T0)
8110#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
8111        /* Load the counter. */
8112        "mv         t0, %[ctr]\n\t"
8113        VL1RE32_V(REG_V16, REG_T0)
8114#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
8115        VSLIDEDOWN_VI(REG_V20, REG_V16, 3)
8116#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
8117        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
8118        VRGATHER_VV(REG_V21, REG_V15, REG_V20)
8119        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
8120        VMV_V_V(REG_V20, REG_V21)
8121#else
8122        VREV8(REG_V20, REG_V20)
8123#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
8124#else
8125        "lw         t3, 12(%[ctr])\n\t"
8126        "slli       t3, t3, 32\n\t"
8127        REV8(REG_T3, REG_T3)
8128#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
8129
8130        /* Load key[0..7]. */
8131        "mv         t0, %[key]\n\t"
8132        VL8RE32_V(REG_V0, REG_T0)
8133        /* Load key[8..11]. */
8134        "addi       t0, t0, 128\n\t"
8135        VL4RE32_V(REG_V8, REG_T0)
8136        /* Load key[12..13]. */
8137        "addi       t0, t0, 64\n\t"
8138        VL2RE32_V(REG_V12, REG_T0)
8139        /* Load last round's key */
8140        "addi       t0, %[key], 224\n\t"
8141        VL1RE32_V(REG_V14, REG_T0)
8142
8143        "beqz       %[sz], L_aes_gcm_256_decrypt_blocks_done\n\t"
8144        "srli       t4, %[sz], 6\n\t"
8145        "beqz       t4, L_aes_gcm_256_decrypt_x4_blocks_done\n\t"
8146
8147        /* Calculate H^[1-4] - GMULT partials */
8148        VMV_V_V(REG_V21, REG_V19)
8149        VMV_V_V(REG_V22, REG_V19)
8150        /* Multiply H * H => H^2 */
8151        VGMUL_VV(REG_V21, REG_V19)
8152        VMV_V_V(REG_V23, REG_V21)
8153        /* Multiply H * H => H^3 */
8154        VGMUL_VV(REG_V22, REG_V21)
8155        /* Multiply H^2 * H^2 => H^4 */
8156        VGMUL_VV(REG_V23, REG_V21)
8157
8158      "L_aes_gcm_256_decrypt_x4_block_loop:\n\t"
8159        /* Load input. */
8160        "mv        t0, %[in]\n\t"
8161        VL4RE32_V(REG_V28, REG_T0)
8162        VMVR_V(REG_V24, REG_V28, 4)
8163        VGMUL_VV(REG_V24, REG_V23)
8164        VGMUL_VV(REG_V25, REG_V22)
8165        VGMUL_VV(REG_V26, REG_V21)
8166        VGMUL_VV(REG_V27, REG_V19)
8167        VXOR_VV(REG_V18, REG_V18, REG_V24)
8168        VXOR_VV(REG_V18, REG_V18, REG_V25)
8169        VXOR_VV(REG_V18, REG_V18, REG_V26)
8170        VXOR_VV(REG_V18, REG_V18, REG_V27)
8171        /* Calculate next 4 counters (+1-4) */
8172#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
8173        VADD_VI(REG_V20, REG_V20, 1)
8174        VMV_V_V(REG_V24, REG_V16)
8175#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
8176        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
8177        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
8178        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
8179#else
8180        VREV8(REG_V17, REG_V20)
8181#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
8182        VSLIDEUP_VI(REG_V24, REG_V17, 3)
8183        VADD_VI(REG_V20, REG_V20, 1)
8184        VMV_V_V(REG_V25, REG_V16)
8185#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
8186        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
8187        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
8188        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
8189#else
8190        VREV8(REG_V17, REG_V20)
8191#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
8192        VSLIDEUP_VI(REG_V25, REG_V17, 3)
8193        VADD_VI(REG_V20, REG_V20, 1)
8194        VMV_V_V(REG_V26, REG_V16)
8195#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
8196        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
8197        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
8198        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
8199#else
8200        VREV8(REG_V17, REG_V20)
8201#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
8202        VSLIDEUP_VI(REG_V26, REG_V17, 3)
8203        VADD_VI(REG_V20, REG_V20, 1)
8204        VMV_V_V(REG_V27, REG_V16)
8205#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
8206        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
8207        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
8208        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
8209#else
8210        VREV8(REG_V17, REG_V20)
8211#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
8212        VSLIDEUP_VI(REG_V27, REG_V17, 3)
8213#else
8214        "addi       t0, t3, 1\n\t"
8215        VMV_V_V(REG_V24, REG_V16)
8216        "addi       t1, t3, 2\n\t"
8217        VMV_V_V(REG_V25, REG_V16)
8218        "slli       t0, t0, 32\n\t"
8219        VMV_V_V(REG_V26, REG_V16)
8220        "slli       t1, t1, 32\n\t"
8221        VMV_V_V(REG_V27, REG_V16)
8222        REV8(REG_T0, REG_T0)
8223        REV8(REG_T1, REG_T1)
8224        VMV_V_X(REG_V20, REG_T0)
8225        "addi       t0, t3, 3\n\t"
8226        VSLIDEUP_VI(REG_V24, REG_V20, 3)
8227        "addi       t3, t3, 4\n\t"
8228        VMV_V_X(REG_V20, REG_T1)
8229        "slli       t0, t0, 32\n\t"
8230        VSLIDEUP_VI(REG_V25, REG_V20, 3)
8231        "slli       t1, t3, 32\n\t"
8232        REV8(REG_T0, REG_T0)
8233        REV8(REG_T1, REG_T1)
8234        VMV_V_X(REG_V20, REG_T0)
8235        VSLIDEUP_VI(REG_V26, REG_V20, 3)
8236        VMV_V_X(REG_V20, REG_T1)
8237        VSLIDEUP_VI(REG_V27, REG_V20, 3)
8238#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
8239
8240        VAESZ_VS(REG_V24, REG_V0)
8241        VAESZ_VS(REG_V25, REG_V0)
8242        VAESZ_VS(REG_V26, REG_V0)
8243        VAESZ_VS(REG_V27, REG_V0)
8244        VAESEM_VS(REG_V24, REG_V1)
8245        VAESEM_VS(REG_V24, REG_V2)
8246        VAESEM_VS(REG_V24, REG_V3)
8247        VAESEM_VS(REG_V24, REG_V4)
8248        VAESEM_VS(REG_V24, REG_V5)
8249        VAESEM_VS(REG_V24, REG_V6)
8250        VAESEM_VS(REG_V24, REG_V7)
8251        VAESEM_VS(REG_V24, REG_V8)
8252        VAESEM_VS(REG_V24, REG_V9)
8253        VAESEM_VS(REG_V24, REG_V10)
8254        VAESEM_VS(REG_V24, REG_V11)
8255        VAESEM_VS(REG_V24, REG_V12)
8256        VAESEM_VS(REG_V24, REG_V13)
8257        VAESEM_VS(REG_V25, REG_V1)
8258        VAESEM_VS(REG_V25, REG_V2)
8259        VAESEM_VS(REG_V25, REG_V3)
8260        VAESEM_VS(REG_V25, REG_V4)
8261        VAESEM_VS(REG_V25, REG_V5)
8262        VAESEM_VS(REG_V25, REG_V6)
8263        VAESEM_VS(REG_V25, REG_V7)
8264        VAESEM_VS(REG_V25, REG_V8)
8265        VAESEM_VS(REG_V25, REG_V9)
8266        VAESEM_VS(REG_V25, REG_V10)
8267        VAESEM_VS(REG_V25, REG_V11)
8268        VAESEM_VS(REG_V25, REG_V12)
8269        VAESEM_VS(REG_V25, REG_V13)
8270        VAESEM_VS(REG_V26, REG_V1)
8271        VAESEM_VS(REG_V26, REG_V2)
8272        VAESEM_VS(REG_V26, REG_V3)
8273        VAESEM_VS(REG_V26, REG_V4)
8274        VAESEM_VS(REG_V26, REG_V5)
8275        VAESEM_VS(REG_V26, REG_V6)
8276        VAESEM_VS(REG_V26, REG_V7)
8277        VAESEM_VS(REG_V26, REG_V8)
8278        VAESEM_VS(REG_V26, REG_V9)
8279        VAESEM_VS(REG_V26, REG_V10)
8280        VAESEM_VS(REG_V26, REG_V11)
8281        VAESEM_VS(REG_V26, REG_V12)
8282        VAESEM_VS(REG_V26, REG_V13)
8283        VAESEM_VS(REG_V27, REG_V1)
8284        VAESEM_VS(REG_V27, REG_V2)
8285        VAESEM_VS(REG_V27, REG_V3)
8286        VAESEM_VS(REG_V27, REG_V4)
8287        VAESEM_VS(REG_V27, REG_V5)
8288        VAESEM_VS(REG_V27, REG_V6)
8289        VAESEM_VS(REG_V27, REG_V7)
8290        VAESEM_VS(REG_V27, REG_V8)
8291        VAESEM_VS(REG_V27, REG_V9)
8292        VAESEM_VS(REG_V27, REG_V10)
8293        VAESEM_VS(REG_V27, REG_V11)
8294        VAESEM_VS(REG_V27, REG_V12)
8295        VAESEM_VS(REG_V27, REG_V13)
8296        VAESEF_VS(REG_V24, REG_V14)
8297        VAESEF_VS(REG_V25, REG_V14)
8298        VAESEF_VS(REG_V26, REG_V14)
8299        VAESEF_VS(REG_V27, REG_V14)
8300        VXOR_VV(REG_V28, REG_V24, REG_V28)
8301        VXOR_VV(REG_V29, REG_V25, REG_V29)
8302        VXOR_VV(REG_V30, REG_V26, REG_V30)
8303        VXOR_VV(REG_V31, REG_V27, REG_V31)
8304        /* Store output. */
8305        "mv         t0, %[out]\n\t"
8306        VS4R_V(REG_V28, REG_T0)
8307        "addi        %[in], %[in], 64\n\t"
8308        "addi        %[out], %[out], 64\n\t"
8309        /* Loop if more elements to process. */
8310        "addi       t4, t4, -1\n\t"
8311        "bnez       t4, L_aes_gcm_256_decrypt_x4_block_loop\n\t"
8312        "andi       %[sz], %[sz], 0x3f\n\t"
8313
8314      "L_aes_gcm_256_decrypt_x4_blocks_done:\n\t"
8315        "srli       t2, %[sz], 4\n\t"
8316        "beqz       t2, L_aes_gcm_256_decrypt_blocks_done\n\t"
8317
8318      "L_aes_gcm_256_decrypt_block_loop:\n\t"
8319#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
8320        VADD_VI(REG_V20, REG_V20, 1)
8321#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
8322        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
8323        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
8324        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
8325#else
8326        VREV8(REG_V17, REG_V20)
8327#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
8328        VMV_V_V(REG_V27, REG_V16)
8329        VSLIDEUP_VI(REG_V27, REG_V17, 3)
8330#else
8331        "addi       t3, t3, 1\n\t"
8332        "slli       t0, t3, 32\n\t"
8333        REV8(REG_T0, REG_T0)
8334        VMV_V_X(REG_V17, REG_T0)
8335        VMV_V_V(REG_V27, REG_V16)
8336        VSLIDEUP_VI(REG_V27, REG_V17, 3)
8337#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
8338
8339        VAESZ_VS(REG_V27, REG_V0)
8340        VAESEM_VS(REG_V27, REG_V1)
8341        VAESEM_VS(REG_V27, REG_V2)
8342        VAESEM_VS(REG_V27, REG_V3)
8343        VAESEM_VS(REG_V27, REG_V4)
8344        VAESEM_VS(REG_V27, REG_V5)
8345        VAESEM_VS(REG_V27, REG_V6)
8346        VAESEM_VS(REG_V27, REG_V7)
8347        VAESEM_VS(REG_V27, REG_V8)
8348        VAESEM_VS(REG_V27, REG_V9)
8349        VAESEM_VS(REG_V27, REG_V10)
8350        VAESEM_VS(REG_V27, REG_V11)
8351        VAESEM_VS(REG_V27, REG_V12)
8352        VAESEM_VS(REG_V27, REG_V13)
8353        VAESEF_VS(REG_V27, REG_V14)
8354
8355        /* Load input. */
8356        "mv         t0, %[in]\n\t"
8357        VL1RE32_V(REG_V17, REG_T0)
8358        VGHSH_VV(REG_V18, REG_V17, REG_V19)
8359        VXOR_VV(REG_V27, REG_V27, REG_V17)
8360        /* Store output. */
8361        "mv         t0, %[out]\n\t"
8362        VS1R_V(REG_V27, REG_T0)
8363
8364        "addi        %[in], %[in], 16\n\t"
8365        "addi        %[out], %[out], 16\n\t"
8366        /* Loop if more elements to process. */
8367        "addi       t2, t2, -1\n\t"
8368        "bnez       t2, L_aes_gcm_256_decrypt_block_loop\n\t"
8369
8370      "L_aes_gcm_256_decrypt_blocks_done:\n\t"
8371        "andi       t2, %[sz], 0xf\n\t"
8372        "beqz       t2, L_aes_gcm_256_decrypt_done\n\t"
8373
8374        VXOR_VV(REG_V17, REG_V17, REG_V17)
8375        "mv         t0, %[scratch]\n\t"
8376        VS1R_V(REG_V17, REG_T0)
8377        "mv         t1, t2\n\t"
8378      "L_aes_gcm_256_decrypt_load_byte:\n\t"
8379        "lb         t0, (%[in])\n\t"
8380        "sb         t0, (%[scratch])\n\t"
8381        "addi       %[in], %[in], 1\n\t"
8382        "addi       %[scratch], %[scratch], 1\n\t"
8383        "addi       t1, t1, -1\n\t"
8384        "bnez       t1, L_aes_gcm_256_decrypt_load_byte\n\t"
8385        "sub        %[scratch], %[scratch], t2\n\t"
8386        "mv         t0, %[scratch]\n\t"
8387        VL1RE32_V(REG_V17, REG_T0)
8388        VGHSH_VV(REG_V18, REG_V17, REG_V19)
8389
8390        /* Encrypt counter for partial block. */
8391#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
8392        VADD_VI(REG_V20, REG_V20, 1)
8393#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
8394        VSETIVLI(REG_X0, 16, 1, 1, 0b000, 0b000)
8395        VRGATHER_VV(REG_V17, REG_V15, REG_V20)
8396        VSETIVLI(REG_X0, 4, 1, 1, 0b010, 0b000)
8397#else
8398        VREV8(REG_V17, REG_V20)
8399#endif /* !WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION */
8400        VMV_V_V(REG_V27, REG_V16)
8401        VSLIDEUP_VI(REG_V27, REG_V17, 3)
8402#else
8403        "addi       t3, t3, 1\n\t"
8404        "slli       t0, t3, 32\n\t"
8405        REV8(REG_T0, REG_T0)
8406        VMV_V_X(REG_V17, REG_T0)
8407        VMV_V_V(REG_V27, REG_V16)
8408        VSLIDEUP_VI(REG_V27, REG_V17, 3)
8409#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
8410
8411        VAESZ_VS(REG_V27, REG_V0)
8412        VAESEM_VS(REG_V27, REG_V1)
8413        VAESEM_VS(REG_V27, REG_V2)
8414        VAESEM_VS(REG_V27, REG_V3)
8415        VAESEM_VS(REG_V27, REG_V4)
8416        VAESEM_VS(REG_V27, REG_V5)
8417        VAESEM_VS(REG_V27, REG_V6)
8418        VAESEM_VS(REG_V27, REG_V7)
8419        VAESEM_VS(REG_V27, REG_V8)
8420        VAESEM_VS(REG_V27, REG_V9)
8421        VAESEM_VS(REG_V27, REG_V10)
8422        VAESEM_VS(REG_V27, REG_V11)
8423        VAESEM_VS(REG_V27, REG_V12)
8424        VAESEM_VS(REG_V27, REG_V13)
8425        VAESEF_VS(REG_V27, REG_V14)
8426
8427        /* Load scratch. */
8428        "mv         t0, %[scratch]\n\t"
8429        VL1RE32_V(REG_V17, REG_T0)
8430        VXOR_VV(REG_V27, REG_V27, REG_V17)
8431        /* Store scratch. */
8432        VS1R_V(REG_V27, REG_T0)
8433        "mv         t1, t2\n\t"
8434      "L_aes_gcm_256_decrypt_store_byte:\n\t"
8435        "lb         t0, (%[scratch])\n\t"
8436        "sb         t0, (%[out])\n\t"
8437        "addi       %[scratch], %[scratch], 1\n\t"
8438        "addi       %[out], %[out], 1\n\t"
8439        "addi       t1, t1, -1\n\t"
8440        "bnez       t1, L_aes_gcm_256_decrypt_store_byte\n\t"
8441        "sub        %[scratch], %[scratch], t2\n\t"
8442
8443      "L_aes_gcm_256_decrypt_done:\n\t"
8444
8445        /* Hash in the lengths of A and C in bits */
8446#ifndef WOLFSSL_RISCV_BASE_BIT_MANIPULATION
8447        /* aSz is only 32-bits */
8448        /* Multiply by 8 do get size in bits. */
8449        "slli       %[aSz], %[aSz], 3\n\t"
8450        "srli       t0, %[aSz], 32\n\t"
8451        "srli       t1, %[aSz], 24\n\t"
8452        "srli       t2, %[aSz], 16\n\t"
8453        "srli       t3, %[aSz], 8\n\t"
8454        /* Top 3 bytes are 0. */
8455        "sh         x0    , 0(%[scratch])\n\t"
8456        "sb         x0    , 2(%[scratch])\n\t"
8457        "sb         t0    , 3(%[scratch])\n\t"
8458        "sb         t1    , 4(%[scratch])\n\t"
8459        "sb         t2    , 5(%[scratch])\n\t"
8460        "sb         t3    , 6(%[scratch])\n\t"
8461        "sb         %[aSz], 7(%[scratch])\n\t"
8462        /* sz is only 32-bits */
8463        /* Multiply by 8 do get size in bits. */
8464        "slli       %[sz], %[sz], 3\n\t"
8465        "srli       t0, %[sz], 32\n\t"
8466        "srli       t1, %[sz], 24\n\t"
8467        "srli       t2, %[sz], 16\n\t"
8468        "srli       t3, %[sz], 8\n\t"
8469        /* Top 3 bytes are 0. */
8470        "sh         x0   ,  8(%[scratch])\n\t"
8471        "sb         x0   , 10(%[scratch])\n\t"
8472        "sb         t0   , 11(%[scratch])\n\t"
8473        "sb         t1   , 12(%[scratch])\n\t"
8474        "sb         t2   , 13(%[scratch])\n\t"
8475        "sb         t3   , 14(%[scratch])\n\t"
8476        "sb         %[sz], 15(%[scratch])\n\t"
8477#else
8478        "slli       t0, %[aSz], 3\n\t"
8479        REV8(REG_T0, REG_T0)
8480        "sd         t0, 0(%[scratch])\n\t"
8481        "slli       t0, %[sz], 3\n\t"
8482        REV8(REG_T0, REG_T0)
8483        "sd         t0, 8(%[scratch])\n\t"
8484#endif /* !WOLFSSL_RISCV_BASE_BIT_MANIPULATION */
8485        "mv         t0, %[scratch]\n\t"
8486        VL1RE32_V(REG_V17, REG_T0)
8487        VGHSH_VV(REG_V18, REG_V17, REG_V19)
8488
8489        VAESZ_VS(REG_V16, REG_V0)
8490        VAESEM_VS(REG_V16, REG_V1)
8491        VAESEM_VS(REG_V16, REG_V2)
8492        VAESEM_VS(REG_V16, REG_V3)
8493        VAESEM_VS(REG_V16, REG_V4)
8494        VAESEM_VS(REG_V16, REG_V5)
8495        VAESEM_VS(REG_V16, REG_V6)
8496        VAESEM_VS(REG_V16, REG_V7)
8497        VAESEM_VS(REG_V16, REG_V8)
8498        VAESEM_VS(REG_V16, REG_V9)
8499        VAESEM_VS(REG_V16, REG_V10)
8500        VAESEM_VS(REG_V16, REG_V11)
8501        VAESEM_VS(REG_V16, REG_V12)
8502        VAESEM_VS(REG_V16, REG_V13)
8503        VAESEF_VS(REG_V16, REG_V14)
8504        VXOR_VV(REG_V18, REG_V18, REG_V16)
8505
8506        "li         t1, 16\n\t"
8507        "blt        %[tagSz], t1, L_aes_gcm_256_decrypt_tag_small\n\t"
8508        "mv         t0, %[tag]\n\t"
8509        VL1RE32_V(REG_V17, REG_T0)
8510        VXOR_VV(REG_V19, REG_V19, REG_V19)
8511        VXOR_VV(REG_V18, REG_V18, REG_V17)
8512        VMSNE_VV(REG_V19, REG_V19, REG_V18)
8513        VCPOP_M(REG_T0, REG_V19)
8514        "beqz       x0, L_aes_gcm_256_decrypt_tag_done\n\t"
8515      "L_aes_gcm_256_decrypt_tag_small:\n\t"
8516        "mv         t0, %[scratch]\n\t"
8517        VS1R_V(REG_V18, REG_T0)
8518        "mv         t1, %[tagSz]\n\t"
8519        "xor        t0, t0, t0\n\t"
8520      "L_aes_gcm_256_decrypt_store_tag_byte:\n\t"
8521        "lb         t2, (%[scratch])\n\t"
8522        "lb         t3, (%[tag])\n\t"
8523        "xor        t3, t3, t2\n\t"
8524        "or         t0, t0, t3\n\t"
8525        "addi       %[scratch], %[scratch], 1\n\t"
8526        "addi       %[tag], %[tag], 1\n\t"
8527        "addi       t1, t1, -1\n\t"
8528        "bnez       t1, L_aes_gcm_256_decrypt_store_tag_byte\n\t"
8529      "L_aes_gcm_256_decrypt_tag_done:\n\t"
8530        "negw       t0, t0\n\t"
8531        "sraiw      t0, t0, 31\n\t"
8532        "andi       %[ret], t0, -180\n\t"
8533
8534        : [out] "+r" (out), [in] "+r" (in), [key] "+r" (key),
8535          [aSz] "+r" (aadSz), [aad] "+r" (aad), [ret] "+r" (ret),
8536          [sz] "+r" (sz)
8537        : [ctr] "r" (ctr), [scratch] "r" (scratch),
8538          [h] "r" (aes->gcm.H), [tag] "r" (tag), [tagSz] "r" (tagSz)
8539#ifndef WOLFSSL_RISCV_VECTOR_BASE_BIT_MANIPULATION
8540          , [rev_idx] "r" (rev_idx)
8541#endif
8542        : "memory", "t0", "t1", "t2", "t3", "t4"
8543    );
8544
8545#ifdef OPENSSL_EXTRA
8546    if ((tag != NULL) && (in != NULL) && (sz != 0)) {
8547        /* store AAD size for next call */
8548        aes->gcm.aadLen = aadSz;
8549    }
8550#endif
8551
8552    return ret;
8553}
8554#endif /* WOLFSSL_AES_256 */
8555
8556/* Decrypt data using AES-GCM.
8557 *
8558 * @param [in]  aes      AES object.
8559 * @param [out] out      Decrypted data.
8560 * @param [in]  in       Data to decrypt and GHASH.
8561 * @param [in]  sz       Number of bytes of data.
8562 * @param [in]  nonce    Nonce used to calculate first IV.
8563 * @param [in]  nonceSz  Length of nonce in bytes.
8564 * @param [out] tag      Authentication tag.
8565 * @param [in]  tagSz    Length of authentication tag in bytes.
8566 * @param [in]  aad      Additional Authentication Data (AAD).
8567 * @param [in]  aadSz    Length of AAD in bytes.
8568 * @return  0 on success.
8569 * @return  BAD_FUNC_ARG when aes, nonce or tag is NULL.
8570 * @return  BAD_FUNC_ARG when nonceSz is zero.
8571 * @return  BAD_FUNC_ARG when aad is NULL but aadSz is not zero.
8572 * @return  BAD_FUNC_ARG when tagSz is less than WOLFSSL_MIN_AUTH_TAG_SZ or
8573 *          greater than WC_AES_BLOCK_SIZE.
8574 * @return  BAD_FUNC_ARG when sz is not zero but in or out is NULL.
8575 * @return  AES_GCM_AUTH_E when authentication tag computed doesn't match
8576 *          tag passed in.
8577 */
8578int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
8579    const byte* nonce, word32 nonceSz, const byte* tag, word32 tagSz,
8580    const byte* aad, word32 aadSz)
8581{
8582    int ret = 0;
8583
8584    /* sanity checks */
8585    if ((aes == NULL) || (nonce == NULL) || (tag == NULL) ||
8586            (tagSz > WC_AES_BLOCK_SIZE) || (tagSz < WOLFSSL_MIN_AUTH_TAG_SZ) ||
8587            ((aad == NULL) && (aadSz > 0)) || (nonceSz == 0) ||
8588            ((sz != 0) && ((in == NULL) || (out == NULL)))) {
8589        WOLFSSL_MSG("a NULL parameter passed in when size is larger than 0");
8590        return BAD_FUNC_ARG;
8591    }
8592
8593    if (ret == 0) {
8594        switch (aes->rounds) {
8595        #ifdef WOLFSSL_AES_128
8596            case 10:
8597                ret = Aes128GcmDecrypt(aes, out, in, sz, nonce, nonceSz, tag,
8598                    tagSz, aad, aadSz);
8599                break;
8600        #endif
8601        #ifdef WOLFSSL_AES_192
8602            case 12:
8603                ret = Aes192GcmDecrypt(aes, out, in, sz, nonce, nonceSz, tag,
8604                    tagSz, aad, aadSz);
8605                break;
8606        #endif
8607        #ifdef WOLFSSL_AES_256
8608            case 14:
8609                ret = Aes256GcmDecrypt(aes, out, in, sz, nonce, nonceSz, tag,
8610                    tagSz, aad, aadSz);
8611                break;
8612        #endif
8613            default:
8614                WOLFSSL_MSG("AES-GCM invalid round number");
8615                ret = BAD_FUNC_ARG;
8616        }
8617    }
8618
8619    return ret;
8620
8621}
8622
8623#endif /* HAVE_AES_DECRYPT */
8624
8625/* END script replace AES-GCM RISC-V 64 with hardware vector crypto */
8626
8627#define HAVE_AES_GCM_ENC_DEC
8628
8629#endif /* !WOLFSSL_RISCV_VECTOR_GCM */
8630
8631#endif /* WOLFSSL_RISCV_VECTOR_CRYPTO_ASM */
8632
8633/* Implement GHASH if we haven't already. */
8634#ifndef HAVE_GHASH
8635/* Remainder values. */
8636static const word16 R[32] = {
8637          0x0000,       0x201c,       0x4038,       0x6024,
8638          0x8070,       0xa06c,       0xc048,       0xe054,
8639          0x00e1,       0x20fd,       0x40d9,       0x60c5,
8640          0x8091,       0xa08d,       0xc0a9,       0xe0b5,
8641
8642          0x0000,       0xc201,       0x8403,       0x4602,
8643          0x0807,       0xca06,       0x8c04,       0x4e05,
8644          0x100e,       0xd20f,       0x940d,       0x560c,
8645          0x1809,       0xda08,       0x9c0a,       0x5e0b,
8646};
8647
8648/* GMULT, multiply in GF2, x and y into x.
8649 *
8650 * @param [in, out]  x  On in, value to GMULT.
8651 *                      On out, result of GMULT.
8652 * @param [in]       y  Value to GMULT.
8653 */
8654static WC_INLINE void GMULT(byte *x, byte m[32][WC_AES_BLOCK_SIZE])
8655{
8656    int i;
8657    word64 z8[2] = {0, 0};
8658    byte a;
8659    word64* x8 = (word64*)x;
8660    word64* m8;
8661    word64 n0, n1, n2, n3;
8662    byte xi;
8663
8664    for (i = 15; i > 0; i--) {
8665        xi = x[i];
8666
8667        /* XOR in (msn * H) */
8668        m8 = (word64*)m[xi & 0xf];
8669        z8[0] ^= m8[0];
8670        z8[1] ^= m8[1];
8671
8672        /* Cache top byte for remainder calculations - lost in rotate. */
8673        a = (byte)(z8[1] >> 56);
8674
8675        /* Rotate Z by 8-bits */
8676        z8[1] = (z8[0] >> 56) | (z8[1] << 8);
8677        z8[0] <<= 8;
8678
8679        /* XOR in (next significant nibble * H) [pre-rotated by 4 bits] */
8680        m8 = (word64*)m[16 + (xi >> 4)];
8681        z8[0] ^= m8[0];
8682        z8[1] ^= m8[1];
8683
8684        /* XOR in (msn * remainder) [pre-rotated by 4 bits] */
8685        z8[0] ^= (word64)R[16 + (a & 0xf)];
8686        /* XOR in next significant nibble (XORed with H) * remainder */
8687        m8 = (word64*)m[xi >> 4];
8688        a ^= (byte)(m8[1] >> 52);
8689        z8[0] ^= (word64)R[a >> 4];
8690    }
8691
8692    xi = x[0];
8693
8694    /* XOR in most significant nibble * H */
8695    m8 = (word64*)m[xi & 0xf];
8696    z8[0] ^= m8[0];
8697    z8[1] ^= m8[1];
8698
8699    /* Cache top byte for remainder calculations - lost in rotate. */
8700    a = (z8[1] >> 56) & 0xf;
8701
8702    /* Rotate z by 4-bits */
8703    n3 = z8[1] & W64LIT(0xf0f0f0f0f0f0f0f0);
8704    n2 = z8[1] & W64LIT(0x0f0f0f0f0f0f0f0f);
8705    n1 = z8[0] & W64LIT(0xf0f0f0f0f0f0f0f0);
8706    n0 = z8[0] & W64LIT(0x0f0f0f0f0f0f0f0f);
8707    z8[1] = (n3 >> 4) | (n2 << 12) | (n0 >> 52);
8708    z8[0] = (n1 >> 4) | (n0 << 12);
8709
8710    /* XOR in next significant nibble * H */
8711    m8 = (word64*)m[xi >> 4];
8712    z8[0] ^= m8[0];
8713    z8[1] ^= m8[1];
8714    /* XOR in most significant nibble * remainder */
8715    z8[0] ^= (word64)R[a];
8716
8717    /* Write back result. */
8718    x8[0] = z8[0];
8719    x8[1] = z8[1];
8720}
8721
8722/* GHASH Additional Authentication Data (AAD) and cipher text.
8723 *
8724 * @param [in]  gcm  GCM object.
8725 * @param [in]  a    Additional Authentication Data (AAD).
8726 * @param [in]  aSz  Size of AAD in bytes.
8727 * @param [in]  c    Cipher text.
8728 * @param [in]  cSz  Size of cipher text in bytes.
8729 * @param [out] s    Hash result.
8730 * @param [in]  sSz  Number of bytes to put into hash result.
8731 */
8732void GHASH(Gcm* gcm, const byte* a, word32 aSz, const byte* c, word32 cSz,
8733    byte* s, word32 sSz)
8734{
8735    if (gcm != NULL) {
8736        byte x[WC_AES_BLOCK_SIZE];
8737        byte scratch[WC_AES_BLOCK_SIZE];
8738        word32 blocks, partial;
8739
8740        XMEMSET(x, 0, WC_AES_BLOCK_SIZE);
8741
8742        /* Hash in A, the Additional Authentication Data */
8743        if (aSz != 0 && a != NULL) {
8744            blocks = aSz / WC_AES_BLOCK_SIZE;
8745            partial = aSz % WC_AES_BLOCK_SIZE;
8746            while (blocks--) {
8747                xorbuf16(x, a);
8748                GMULT(x, gcm->M0);
8749                a += WC_AES_BLOCK_SIZE;
8750            }
8751            if (partial != 0) {
8752                XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
8753                XMEMCPY(scratch, a, partial);
8754                xorbuf16(x, scratch);
8755                GMULT(x, gcm->M0);
8756            }
8757        }
8758
8759        /* Hash in C, the Ciphertext */
8760        if (cSz != 0 && c != NULL) {
8761            blocks = cSz / WC_AES_BLOCK_SIZE;
8762            partial = cSz % WC_AES_BLOCK_SIZE;
8763            while (blocks--) {
8764                xorbuf16(x, c);
8765                GMULT(x, gcm->M0);
8766                c += WC_AES_BLOCK_SIZE;
8767            }
8768            if (partial != 0) {
8769                XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
8770                XMEMCPY(scratch, c, partial);
8771                xorbuf16(x, scratch);
8772                GMULT(x, gcm->M0);
8773            }
8774        }
8775
8776        /* Hash in the lengths of A and C in bits */
8777        FlattenSzInBits(&scratch[0], aSz);
8778        FlattenSzInBits(&scratch[8], cSz);
8779        xorbuf16(x, scratch);
8780        GMULT(x, gcm->M0);
8781
8782        /* Copy the result into s. */
8783        XMEMCPY(s, x, sSz);
8784    }
8785}
8786#endif /* !HAVE_GHASH */
8787
8788#ifndef HAVE_AES_GCM_ENC_DEC
8789/* Increment AES-GCM counter.
8790 *
8791 * Big-endian byte ordering.
8792 *
8793 * @param [in, out] inOutCtr  Counter value to be incremented.
8794 */
8795static WC_INLINE void IncrementGcmCounter(byte* inOutCtr)
8796{
8797    int i;
8798
8799    /* Big-endian array - start at last element and move back. */
8800    for (i = WC_AES_BLOCK_SIZE - 1; i >= WC_AES_BLOCK_SIZE - CTR_SZ; i--) {
8801        /* Result not zero means no carry. */
8802        if ((++inOutCtr[i]) != 0) {
8803            return;
8804        }
8805    }
8806}
8807
8808/* Encrypt data using AES-GCM.
8809 *
8810 * @param [in]  aes      AES object.
8811 * @param [out] out      Encrypted data.
8812 * @param [in]  in       Data to encrypt.
8813 * @param [in]  sz       Number of bytes of data.
8814 * @param [in]  nonce    Nonce used to calculate first IV.
8815 * @param [in]  nonceSz  Length of nonce in bytes.
8816 * @param [out] tag      Authentication tag.
8817 * @param [in]  tagSz    Length of authentication tag in bytes.
8818 * @param [in]  aad      Additional Authentication Data (AAD).
8819 * @param [in]  aadSz    Length of AAD in bytes.
8820 * @return  0 on success.
8821 * @return  BAD_FUNC_ARG when aes, nonce or tag is NULL.
8822 * @return  BAD_FUNC_ARG when nonceSz is zero.
8823 * @return  BAD_FUNC_ARG when aad is NULL but aadSz is not zero.
8824 * @return  BAD_FUNC_ARG when tagSz is less than WOLFSSL_MIN_AUTH_TAG_SZ or
8825 *          greater than WC_AES_BLOCK_SIZE.
8826 * @return  BAD_FUNC_ARG when sz is not zero but in or out is NULL.
8827 */
8828int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz,
8829    const byte* nonce, word32 nonceSz, byte* tag, word32 tagSz,
8830    const byte* aad, word32 aadSz)
8831{
8832    int ret = 0;
8833    word32 blocks = sz / WC_AES_BLOCK_SIZE;
8834    word32 partial = sz % WC_AES_BLOCK_SIZE;
8835    const byte* p = in;
8836    byte* c = out;
8837    ALIGN16 byte counter[WC_AES_BLOCK_SIZE];
8838    ALIGN16 byte initialCounter[WC_AES_BLOCK_SIZE];
8839    ALIGN16 byte scratch[WC_AES_BLOCK_SIZE];
8840
8841    /* Validate parameters. */
8842    if ((aes == NULL) || (nonce == NULL) || (nonceSz == 0) || (tag == NULL) ||
8843            ((aad == NULL) && (aadSz > 0)) || ((sz != 0) && ((in == NULL) ||
8844            (out == NULL)))) {
8845        WOLFSSL_MSG("a NULL parameter passed in when size is larger than 0");
8846        ret = BAD_FUNC_ARG;
8847    }
8848
8849    if ((ret == 0) && ((tagSz < WOLFSSL_MIN_AUTH_TAG_SZ) ||
8850            (tagSz > WC_AES_BLOCK_SIZE))) {
8851        WOLFSSL_MSG("GcmEncrypt tagSz error");
8852        ret = BAD_FUNC_ARG;
8853    }
8854
8855
8856    if (ret == 0) {
8857        if (nonceSz == GCM_NONCE_MID_SZ) {
8858            /* Counter is IV with bottom 4 bytes set to: 0x00,0x00,0x00,0x01. */
8859            XMEMCPY(counter, nonce, nonceSz);
8860            XMEMSET(counter + GCM_NONCE_MID_SZ, 0,
8861                WC_AES_BLOCK_SIZE - GCM_NONCE_MID_SZ - 1);
8862            counter[WC_AES_BLOCK_SIZE - 1] = 1;
8863        }
8864        else {
8865            /* Counter is GHASH of IV. */
8866        #ifdef OPENSSL_EXTRA
8867            word32 aadTemp = aes->gcm.aadLen;
8868            aes->gcm.aadLen = 0;
8869        #endif
8870            GHASH(&aes->gcm, NULL, 0, nonce, nonceSz, counter, WC_AES_BLOCK_SIZE);
8871        #ifdef OPENSSL_EXTRA
8872            aes->gcm.aadLen = aadTemp;
8873        #endif
8874        }
8875        memcpy16(initialCounter, counter);
8876
8877        while (blocks--) {
8878            IncrementGcmCounter(counter);
8879            wc_AesEncrypt(aes, counter, scratch);
8880            xorbufout16(c, scratch, p);
8881            p += WC_AES_BLOCK_SIZE;
8882            c += WC_AES_BLOCK_SIZE;
8883        }
8884
8885        if (partial != 0) {
8886            IncrementGcmCounter(counter);
8887            wc_AesEncrypt(aes, counter, scratch);
8888            xorbufout(c, scratch, p, partial);
8889        }
8890        if (tag) {
8891            GHASH(&aes->gcm, aad, aadSz, out, sz, tag, tagSz);
8892            wc_AesEncrypt(aes, initialCounter, scratch);
8893            xorbuf(tag, scratch, tagSz);
8894        #ifdef OPENSSL_EXTRA
8895            if (!in && !sz)
8896                /* store AAD size for next call */
8897                aes->gcm.aadLen = aadSz;
8898        #endif
8899        }
8900    }
8901
8902    return ret;
8903}
8904
8905#ifdef HAVE_AES_DECRYPT
8906/* Decrypt data using AES-GCM.
8907 *
8908 * @param [in]  aes      AES object.
8909 * @param [out] out      Decrypted data.
8910 * @param [in]  in       Data to decrypt and GHASH.
8911 * @param [in]  sz       Number of bytes of data.
8912 * @param [in]  nonce    Nonce used to calculate first IV.
8913 * @param [in]  nonceSz  Length of nonce in bytes.
8914 * @param [out] tag      Authentication tag.
8915 * @param [in]  tagSz    Length of authentication tag in bytes.
8916 * @param [in]  aad      Additional Authentication Data (AAD).
8917 * @param [in]  aadSz    Length of AAD in bytes.
8918 * @return  0 on success.
8919 * @return  BAD_FUNC_ARG when aes, nonce or tag is NULL.
8920 * @return  BAD_FUNC_ARG when nonceSz is zero.
8921 * @return  BAD_FUNC_ARG when aad is NULL but aadSz is not zero.
8922 * @return  BAD_FUNC_ARG when tagSz is less than WOLFSSL_MIN_AUTH_TAG_SZ or
8923 *          greater than WC_AES_BLOCK_SIZE.
8924 * @return  BAD_FUNC_ARG when sz is not zero but in or out is NULL.
8925 * @return  AES_GCM_AUTH_E when authentication tag computed doesn't match
8926 *          tag passed in.
8927 */
8928int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
8929    const byte* nonce, word32 nonceSz, const byte* tag, word32 tagSz,
8930    const byte* aad, word32 aadSz)
8931{
8932    int ret = 0;
8933    word32 blocks = sz / WC_AES_BLOCK_SIZE;
8934    word32 partial = sz % WC_AES_BLOCK_SIZE;
8935    const byte* c = in;
8936    byte* p = out;
8937    ALIGN16 byte counter[WC_AES_BLOCK_SIZE];
8938    ALIGN16 byte scratch[WC_AES_BLOCK_SIZE];
8939    ALIGN16 byte Tprime[WC_AES_BLOCK_SIZE];
8940    ALIGN16 byte EKY0[WC_AES_BLOCK_SIZE];
8941    sword32 res;
8942
8943    /* Validate parameters. */
8944    if ((aes == NULL) || (nonce == NULL) || (tag == NULL) ||
8945            (tagSz > WC_AES_BLOCK_SIZE) || (tagSz < WOLFSSL_MIN_AUTH_TAG_SZ) ||
8946            ((aad == NULL) && (aadSz > 0)) || (nonceSz == 0) ||
8947            ((sz != 0) && ((in == NULL) || (out == NULL)))) {
8948        WOLFSSL_MSG("a NULL parameter passed in when size is larger than 0");
8949        ret = BAD_FUNC_ARG;
8950    }
8951
8952    if (ret == 0) {
8953        if (nonceSz == GCM_NONCE_MID_SZ) {
8954            /* Counter is IV with bottom 4 bytes set to: 0x00,0x00,0x00,0x01. */
8955            XMEMCPY(counter, nonce, nonceSz);
8956            XMEMSET(counter + GCM_NONCE_MID_SZ, 0,
8957                WC_AES_BLOCK_SIZE - GCM_NONCE_MID_SZ - 1);
8958            counter[WC_AES_BLOCK_SIZE - 1] = 1;
8959        }
8960        else {
8961            /* Counter is GHASH of IV. */
8962        #ifdef OPENSSL_EXTRA
8963            word32 aadTemp = aes->gcm.aadLen;
8964            aes->gcm.aadLen = 0;
8965        #endif
8966            GHASH(&aes->gcm, NULL, 0, nonce, nonceSz, counter, WC_AES_BLOCK_SIZE);
8967        #ifdef OPENSSL_EXTRA
8968            aes->gcm.aadLen = aadTemp;
8969        #endif
8970        }
8971
8972        /* Calc the tag again using received auth data and the cipher text */
8973        GHASH(&aes->gcm, aad, aadSz, in, sz, Tprime, sizeof(Tprime));
8974        wc_AesEncrypt(aes, counter, EKY0);
8975        xorbuf(Tprime, EKY0, sizeof(Tprime));
8976    #ifdef WC_AES_GCM_DEC_AUTH_EARLY
8977        /* ConstantCompare returns the cumulative bitwise or of the bitwise xor
8978         * of the pairwise bytes in the strings.
8979         */
8980        res = ConstantCompare(tag, Tprime, tagSz);
8981        /* convert positive retval from ConstantCompare() to all-1s word, in
8982         * constant time.
8983         */
8984        res = 0 - (sword32)(((word32)(0 - res)) >> 31U);
8985        ret = res & AES_GCM_AUTH_E;
8986    }
8987    if (ret == 0) {
8988    #endif
8989
8990    #ifdef OPENSSL_EXTRA
8991        if (!out) {
8992            /* authenticated, non-confidential data */
8993            /* store AAD size for next call */
8994            aes->gcm.aadLen = aadSz;
8995        }
8996    #endif
8997
8998        while (blocks--) {
8999            IncrementGcmCounter(counter);
9000            wc_AesEncrypt(aes, counter, scratch);
9001            xorbufout16(p, scratch, c);
9002            p += WC_AES_BLOCK_SIZE;
9003            c += WC_AES_BLOCK_SIZE;
9004        }
9005
9006        if (partial != 0) {
9007            IncrementGcmCounter(counter);
9008            wc_AesEncrypt(aes, counter, scratch);
9009            xorbuf(scratch, c, partial);
9010            XMEMCPY(p, scratch, partial);
9011        }
9012
9013    #ifndef WC_AES_GCM_DEC_AUTH_EARLY
9014        /* ConstantCompare returns the cumulative bitwise or of the bitwise xor
9015         * of the pairwise bytes in the strings.
9016         */
9017        res = ConstantCompare(tag, Tprime, (int)tagSz);
9018        /* convert positive retval from ConstantCompare() to all-1s word, in
9019         * constant time.
9020         */
9021        res = 0 - (sword32)(((word32)(0 - res)) >> 31U);
9022        /* now use res as a mask for constant time return of ret, unless tag
9023         * mismatch, whereupon AES_GCM_AUTH_E is returned.
9024         */
9025        ret = (ret & ~res) | (res & AES_GCM_AUTH_E);
9026    #endif
9027    }
9028
9029    return ret;
9030}
9031#endif /* HAVE_AES_DECRYPT */
9032#endif /* !HAVE_AES_GCM_ENC_DEC */
9033
9034#endif /* HAVE_AESGCM */
9035
9036#ifdef HAVE_AESCCM
9037
9038static void roll_x(Aes* aes, const byte* in, word32 inSz, byte* out)
9039{
9040    /* process the bulk of the data */
9041    while (inSz >= WC_AES_BLOCK_SIZE) {
9042        xorbuf16(out, in);
9043        in += WC_AES_BLOCK_SIZE;
9044        inSz -= WC_AES_BLOCK_SIZE;
9045
9046        wc_AesEncrypt(aes, out, out);
9047    }
9048
9049    /* process remainder of the data */
9050    if (inSz > 0) {
9051        xorbuf(out, in, inSz);
9052        wc_AesEncrypt(aes, out, out);
9053    }
9054}
9055
9056
9057static void roll_auth(Aes* aes, const byte* in, word32 inSz, byte* out)
9058{
9059    word32 authLenSz;
9060    word32 remainder;
9061
9062    /* encode the length in */
9063    if (inSz <= 0xFEFF) {
9064        authLenSz = 2;
9065        out[0] ^= ((inSz & 0xFF00) >> 8);
9066        out[1] ^=  (inSz & 0x00FF);
9067    }
9068    else {
9069        authLenSz = 6;
9070        out[0] ^= 0xFF; out[1] ^= 0xFE;
9071        out[2] ^= ((inSz & 0xFF000000) >> 24);
9072        out[3] ^= ((inSz & 0x00FF0000) >> 16);
9073        out[4] ^= ((inSz & 0x0000FF00) >>  8);
9074        out[5] ^=  (inSz & 0x000000FF);
9075    }
9076    /* Note, the protocol handles auth data up to 2^64, but we are
9077     * using 32-bit sizes right now, so the bigger data isn't handled.
9078     */
9079
9080    /* start fill out the rest of the first block */
9081    remainder = WC_AES_BLOCK_SIZE - authLenSz;
9082    if (inSz >= remainder) {
9083        /* plenty of bulk data to fill the remainder of this block */
9084        xorbuf(out + authLenSz, in, remainder);
9085        inSz -= remainder;
9086        in += remainder;
9087    }
9088    else {
9089        /* not enough bulk data, copy what is available, and pad zero */
9090        xorbuf(out + authLenSz, in, inSz);
9091        inSz = 0;
9092    }
9093    wc_AesEncrypt(aes, out, out);
9094
9095    if (inSz > 0)
9096        roll_x(aes, in, inSz, out);
9097}
9098
9099
9100static WC_INLINE void AesCcmCtrInc(byte* B, word32 lenSz)
9101{
9102    word32 i;
9103
9104    for (i = 0; i < lenSz; i++) {
9105        if (++B[WC_AES_BLOCK_SIZE - 1 - i] != 0) return;
9106    }
9107}
9108
9109/* return 0 on success */
9110int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
9111                   const byte* nonce, word32 nonceSz,
9112                   byte* authTag, word32 authTagSz,
9113                   const byte* authIn, word32 authInSz)
9114{
9115    int ret = 0;
9116
9117    /* sanity check on arguments */
9118    if ((aes == NULL) || ((inSz != 0) && ((in == NULL) || (out == NULL))) ||
9119            (nonce == NULL) || (authTag == NULL) || (nonceSz < 7) ||
9120            (nonceSz > 13)) {
9121        ret = BAD_FUNC_ARG;
9122    }
9123
9124    if ((ret == 0) && (wc_AesCcmCheckTagSize(authTagSz) != 0)) {
9125        ret = BAD_FUNC_ARG;
9126    }
9127
9128    if (ret == 0) {
9129        byte A[WC_AES_BLOCK_SIZE];
9130        byte B[WC_AES_BLOCK_SIZE];
9131        byte lenSz;
9132        byte i;
9133
9134        XMEMCPY(B+1, nonce, nonceSz);
9135        lenSz = WC_AES_BLOCK_SIZE - 1 - (byte)nonceSz;
9136        B[0] = (authInSz > 0 ? 64 : 0)
9137             + (8 * (((byte)authTagSz - 2) / 2))
9138             + (lenSz - 1);
9139        for (i = 0; (i < lenSz) && (i < (byte)sizeof(word32)); i++) {
9140            B[WC_AES_BLOCK_SIZE - 1 - i] = inSz >> (8 * i);
9141        }
9142        for (; i < lenSz; i++) {
9143            B[WC_AES_BLOCK_SIZE - 1 - i] = 0;
9144        }
9145
9146        wc_AesEncrypt(aes, B, A);
9147
9148        if (authInSz > 0) {
9149            roll_auth(aes, authIn, authInSz, A);
9150        }
9151        if (inSz > 0) {
9152            roll_x(aes, in, inSz, A);
9153        }
9154        XMEMCPY(authTag, A, authTagSz);
9155
9156        B[0] = lenSz - 1;
9157        for (i = 0; i < lenSz; i++) {
9158            B[WC_AES_BLOCK_SIZE - 1 - i] = 0;
9159        }
9160        wc_AesEncrypt(aes, B, A);
9161        xorbuf(authTag, A, authTagSz);
9162
9163        B[15] = 1;
9164        while (inSz >= WC_AES_BLOCK_SIZE) {
9165            wc_AesEncrypt(aes, B, A);
9166            xorbuf16(A, in);
9167            memcpy16(out, A);
9168
9169            AesCcmCtrInc(B, lenSz);
9170            inSz -= WC_AES_BLOCK_SIZE;
9171            in += WC_AES_BLOCK_SIZE;
9172            out += WC_AES_BLOCK_SIZE;
9173        }
9174        if (inSz > 0) {
9175            wc_AesEncrypt(aes, B, A);
9176            xorbuf(A, in, inSz);
9177            XMEMCPY(out, A, inSz);
9178        }
9179
9180        ForceZero(A, WC_AES_BLOCK_SIZE);
9181        ForceZero(B, WC_AES_BLOCK_SIZE);
9182    }
9183
9184    return ret;
9185}
9186
9187#ifdef HAVE_AES_DECRYPT
9188int  wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
9189                   const byte* nonce, word32 nonceSz,
9190                   const byte* authTag, word32 authTagSz,
9191                   const byte* authIn, word32 authInSz)
9192{
9193    int ret = 0;
9194
9195    /* sanity check on arguments */
9196    if ((aes == NULL) || ((inSz != 0) && ((in == NULL) || (out == NULL))) ||
9197            (nonce == NULL) || (authTag == NULL) || (nonceSz < 7) ||
9198            (nonceSz > 13)) {
9199        ret = BAD_FUNC_ARG;
9200    }
9201
9202    if ((ret == 0) && (wc_AesCcmCheckTagSize(authTagSz) != 0)) {
9203        ret = BAD_FUNC_ARG;
9204    }
9205
9206    if (ret == 0) {
9207        byte A[WC_AES_BLOCK_SIZE];
9208        byte B[WC_AES_BLOCK_SIZE];
9209        byte lenSz;
9210        byte i;
9211        byte* o = out;
9212        word32 oSz = inSz;
9213
9214        XMEMCPY(B+1, nonce, nonceSz);
9215        lenSz = WC_AES_BLOCK_SIZE - 1 - (byte)nonceSz;
9216
9217        B[0] = lenSz - 1;
9218        for (i = 0; i < lenSz; i++) {
9219            B[WC_AES_BLOCK_SIZE - 1 - i] = 0;
9220        }
9221        B[15] = 1;
9222
9223        while (oSz >= WC_AES_BLOCK_SIZE) {
9224            wc_AesEncrypt(aes, B, A);
9225            xorbuf16(A, in);
9226            memcpy16(o, A);
9227
9228            AesCcmCtrInc(B, lenSz);
9229            oSz -= WC_AES_BLOCK_SIZE;
9230            in += WC_AES_BLOCK_SIZE;
9231            o += WC_AES_BLOCK_SIZE;
9232        }
9233        if (inSz > 0) {
9234            wc_AesEncrypt(aes, B, A);
9235            xorbuf(A, in, oSz);
9236            XMEMCPY(o, A, oSz);
9237        }
9238
9239        for (i = 0; i < lenSz; i++) {
9240            B[WC_AES_BLOCK_SIZE - 1 - i] = 0;
9241        }
9242        wc_AesEncrypt(aes, B, A);
9243
9244        B[0] = (authInSz > 0 ? 64 : 0)
9245             + (8 * (((byte)authTagSz - 2) / 2))
9246             + (lenSz - 1);
9247        for (i = 0; (i < lenSz) && (i < (byte)sizeof(word32)); i++) {
9248            B[WC_AES_BLOCK_SIZE - 1 - i] = inSz >> (8 * i);
9249        }
9250        for (; i < lenSz; i++) {
9251            B[WC_AES_BLOCK_SIZE - 1 - i] = 0;
9252        }
9253
9254        wc_AesEncrypt(aes, B, A);
9255
9256        if (authInSz > 0) {
9257            roll_auth(aes, authIn, authInSz, A);
9258        }
9259        if (inSz > 0) {
9260            roll_x(aes, out, inSz, A);
9261        }
9262
9263        B[0] = lenSz - 1;
9264        for (i = 0; i < lenSz; i++) {
9265            B[WC_AES_BLOCK_SIZE - 1 - i] = 0;
9266        }
9267        wc_AesEncrypt(aes, B, B);
9268        xorbuf(A, B, authTagSz);
9269
9270        if (ConstantCompare(A, authTag, authTagSz) != 0) {
9271            /* If the authTag check fails, don't keep the decrypted data.
9272             * Unfortunately, you need the decrypted data to calculate the
9273             * check value. */
9274            XMEMSET(out, 0, inSz);
9275            ret = AES_CCM_AUTH_E;
9276        }
9277
9278        ForceZero(A, WC_AES_BLOCK_SIZE);
9279        ForceZero(B, WC_AES_BLOCK_SIZE);
9280        o = NULL;
9281    }
9282
9283    return ret;
9284}
9285#endif /* HAVE_AES_DECRYPT */
9286#endif /* HAVE_AESCCM */
9287
9288#endif /* WOLFSSL_RISCV_ASM */
9289
9290#endif /* !NO_AES */
9291