luna - wolfssl/wolfcrypt/src/aes.c

    1/* aes.c
    2 *
    3 * Copyright (C) 2006-2026 wolfSSL Inc.
    4 *
    5 * This file is part of wolfSSL.
    6 *
    7 * wolfSSL is free software; you can redistribute it and/or modify
    8 * it under the terms of the GNU General Public License as published by
    9 * the Free Software Foundation; either version 3 of the License, or
   10 * (at your option) any later version.
   11 *
   12 * wolfSSL is distributed in the hope that it will be useful,
   13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
   14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   15 * GNU General Public License for more details.
   16 *
   17 * You should have received a copy of the GNU General Public License
   18 * along with this program; if not, write to the Free Software
   19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
   20 */
   21
   22/*
   23
   24DESCRIPTION
   25This library provides the interfaces to the Advanced Encryption Standard (AES)
   26for encrypting and decrypting data. AES is the standard known for a symmetric
   27block cipher mechanism that uses n-bit binary string parameter key with 128-bits,
   28192-bits, and 256-bits of key sizes.
   29
   30*/
   31
   32/*
   33 * AES Build Options:
   34 *
   35 * Core:
   36 * NO_AES:                  Disable AES support entirely          default: off
   37 * WOLFSSL_AES_128:         Enable AES-128 key size               default: on
   38 * WOLFSSL_AES_192:         Enable AES-192 key size               default: on
   39 * WOLFSSL_AES_256:         Enable AES-256 key size               default: on
   40 * AES_MAX_KEY_SIZE:        Maximum AES key size in bits           default: 256
   41 *
   42 * Cipher Modes:
   43 * HAVE_AES_CBC:            Enable AES-CBC mode                   default: on
   44 * HAVE_AES_ECB:            Enable AES-ECB mode                   default: off
   45 * HAVE_AES_DECRYPT:        Enable AES decryption                 default: on
   46 * WOLFSSL_AES_COUNTER:     Enable AES-CTR mode                   default: off
   47 * WOLFSSL_AES_CFB:         Enable AES-CFB mode                   default: off
   48 * WOLFSSL_NO_AES_CFB_1_8:  Disable AES-CFB-1 and AES-CFB-8      default: off
   49 * WOLFSSL_AES_OFB:         Enable AES-OFB mode                   default: off
   50 * WOLFSSL_AES_DIRECT:      Enable direct AES encrypt/decrypt API default: off
   51 * WOLFSSL_AES_XTS:         Enable AES-XTS mode                   default: off
   52 * WOLFSSL_AES_CTS:         Enable AES-CTS (ciphertext stealing)  default: off
   53 * WOLFSSL_AES_SIV:         Enable AES-SIV (synthetic IV) mode    default: off
   54 * WOLFSSL_AES_EAX:         Enable AES-EAX AEAD mode              default: off
   55 * WOLFSSL_CMAC:            Enable AES-CMAC (RFC 4493)            default: off
   56 * HAVE_AESCCM:             Enable AES-CCM mode                   default: off
   57 * HAVE_AES_KEYWRAP:        Enable AES key wrap (RFC 3394)        default: off
   58 * WOLFSSL_AES_CBC_LENGTH_CHECKS: Validate CBC input length       default: off
   59 *
   60 * AES-GCM:
   61 * HAVE_AESGCM:             Enable AES-GCM mode                   default: off
   62 * HAVE_AESGCM_DECRYPT:     Enable AES-GCM decryption             default: on
   63 *                           (when HAVE_AESGCM is enabled)
   64 * WOLFSSL_AESGCM_STREAM:   Enable streaming AES-GCM API          default: off
   65 * WC_AES_GCM_DEC_AUTH_EARLY: Authenticate tag before decryption  default: off
   66 * GCM_SMALL:               Small GCM table, saves memory         default: off
   67 * GCM_TABLE:               Full 4-bit GCM lookup table, faster   default: off
   68 * GCM_TABLE_4BIT:          Explicit 4-bit GCM table mode         default: off
   69 * GCM_WORD32:              Use 32-bit word GCM implementation    default: off
   70 * GCM_GMULT_LEN:           GCM GMULT length optimization         default: off
   71 *
   72 * AES-XTS Stream:
   73 * WOLFSSL_AESXTS_STREAM:   Enable streaming AES-XTS API          default: off
   74 * WC_AESXTS_STREAM_NO_REQUEST_ACCOUNTING:
   75 *                           Disable XTS stream request accounting default: off
   76 * WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS:
   77 *                           Support both encrypt and decrypt keys default: off
   78 *                           simultaneously in XTS context
   79 *
   80 * Performance / Side-Channel:
   81 * WOLFSSL_AESNI:           Enable Intel AES-NI instructions      default: off
   82 * WOLFSSL_AESNI_BY4:       AES-NI 4-block parallel processing    default: off
   83 * WOLFSSL_AESNI_BY6:       AES-NI 6-block parallel processing    default: off
   84 * USE_INTEL_SPEEDUP:       Intel AVX/AVX2 for AES acceleration   default: off
   85 * WOLFSSL_AES_SMALL_TABLES: Use smaller AES S-box tables         default: off
   86 * WOLFSSL_AES_NO_UNROLL:   Disable AES round loop unrolling      default: off
   87 * WOLFSSL_AES_TOUCH_LINES: Touch all cache lines for             default: off
   88 *                           side-channel resistance
   89 * WC_AES_BITSLICED:        Use bitsliced AES implementation      default: off
   90 * AES_GCM_GMULT_NCT:       GCM GMULT non-constant-time          default: off
   91 * NO_WOLFSSL_ALLOC_ALIGN:  Disable aligned memory allocation     default: off
   92 *
   93 * Hardware Acceleration (AES-specific):
   94 * WC_ASYNC_ENABLE_AES:     Enable async AES operations           default: off
   95 * WOLFSSL_CRYPTOCELL_AES:  CryptoCell AES acceleration           default: off
   96 * WOLFSSL_DEVCRYPTO_AES:   /dev/crypto AES acceleration          default: off
   97 * WOLFSSL_DEVCRYPTO_CBC:   /dev/crypto AES-CBC acceleration      default: off
   98 * WOLFSSL_KCAPI_AES:       Linux kernel crypto API for AES       default: off
   99 * WOLFSSL_NO_KCAPI_AES_CBC: Disable KCAPI AES-CBC                default: off
  100 * WOLFSSL_NRF51_AES:       nRF51 hardware AES                    default: off
  101 * WOLFSSL_PSA_NO_AES:      Disable PSA AES                       default: off
  102 * WOLFSSL_SCE_NO_AES:      Disable Renesas SCE AES               default: off
  103 * NO_IMX6_CAAM_AES:        Disable i.MX6 CAAM AES               default: off
  104 * WOLFSSL_AFALG_XILINX_AES: AF_ALG Xilinx AES acceleration      default: off
  105 * NO_WOLFSSL_ESP32_CRYPT_AES: Disable ESP32 AES acceleration     default: off
  106 * STM32_CRYPTO_AES_ONLY:   STM32 AES-only crypto mode            default: off
  107 *
  108 * Debug:
  109 * WC_DEBUG_CIPHER_LIFECYCLE: Debug cipher init/free lifecycle     default: off
  110 * WOLFSSL_HW_METRICS:      Track hardware acceleration usage     default: off
  111 */
  112
  113#include <wolfssl/wolfcrypt/libwolfssl_sources.h>
  114
  115#if !defined(NO_AES)
  116
  117/* Tip: Locate the software cipher modes by searching for "Software AES" */
  118
  119#if FIPS_VERSION3_GE(2,0,0)
  120    /* set NO_WRAPPERS before headers, use direct internal f()s not wrappers */
  121    #define FIPS_NO_WRAPPERS
  122
  123    #ifdef USE_WINDOWS_API
  124        #pragma code_seg(".fipsA$b")
  125        #pragma const_seg(".fipsB$b")
  126    #endif
  127#endif
  128
  129#include <wolfssl/wolfcrypt/aes.h>
  130
  131#ifdef WOLFSSL_AESNI
  132#include <wmmintrin.h>
  133#include <emmintrin.h>
  134#include <smmintrin.h>
  135#endif /* WOLFSSL_AESNI */
  136
  137#include <wolfssl/wolfcrypt/cpuid.h>
  138
  139#ifdef WOLF_CRYPTO_CB
  140    #include <wolfssl/wolfcrypt/cryptocb.h>
  141#endif
  142
  143#ifdef WOLFSSL_NXP_HASHCRYPT_AES
  144    #include <wolfssl/wolfcrypt/port/nxp/hashcrypt_port.h>
  145#endif
  146
  147#ifdef WOLFSSL_SECO_CAAM
  148#include <wolfssl/wolfcrypt/port/caam/wolfcaam.h>
  149#endif
  150
  151#ifdef WOLFSSL_IMXRT_DCP
  152    #include <wolfssl/wolfcrypt/port/nxp/dcp_port.h>
  153#endif
  154#if defined(WOLFSSL_SE050) && defined(WOLFSSL_SE050_CRYPT)
  155    #include <wolfssl/wolfcrypt/port/nxp/se050_port.h>
  156#endif
  157#ifdef WOLFSSL_MICROCHIP_TA100
  158    #include <wolfssl/wolfcrypt/port/atmel/atmel.h>
  159#endif
  160#ifdef WOLFSSL_CMAC
  161    #include <wolfssl/wolfcrypt/cmac.h>
  162#endif
  163
  164#if defined(WOLFSSL_HAVE_PSA) && !defined(WOLFSSL_PSA_NO_AES)
  165    #include <wolfssl/wolfcrypt/port/psa/psa.h>
  166#endif
  167
  168#if defined(WOLFSSL_MAX3266X) || defined(WOLFSSL_MAX3266X_OLD)
  169    #include <wolfssl/wolfcrypt/port/maxim/max3266x.h>
  170#ifdef MAX3266X_CB
  171    /* Revert back to SW so HW CB works */
  172    /* HW only works for AES: ECB, CBC, and partial via ECB for other modes */
  173    #include <wolfssl/wolfcrypt/port/maxim/max3266x-cryptocb.h>
  174    /* Turn off MAX3266X_AES in the context of this file when using CB */
  175    #undef MAX3266X_AES
  176#endif
  177#endif
  178
  179#if defined(WOLFSSL_TI_CRYPT)
  180    #include <wolfcrypt/src/port/ti/ti-aes.c>
  181
  182    #define AesEncrypt_preFetchOpt(aes, inBlock, outBlock, do_preFetch) \
  183        wc_AesEncryptDirect(aes, outBlock, inBlock)
  184    #define AesDecrypt_preFetchOpt(aes, inBlock, outBlock, do_preFetch) \
  185        wc_AesDecryptDirect(aes, outBlock, inBlock)
  186#else
  187
  188
  189#if defined(WOLFSSL_PSOC6_CRYPTO)
  190    #include <wolfssl/wolfcrypt/port/cypress/psoc6_crypto.h>
  191#endif /* WOLFSSL_PSOC6_CRYPTO */
  192
  193#ifdef NO_INLINE
  194    #include <wolfssl/wolfcrypt/misc.h>
  195#else
  196    #define WOLFSSL_MISC_INCLUDED
  197    #include <wolfcrypt/src/misc.c>
  198#endif
  199
  200#if !defined(WOLFSSL_RISCV_ASM)
  201
  202#ifdef WOLFSSL_IMX6_CAAM_BLOB
  203    /* case of possibly not using hardware acceleration for AES but using key
  204       blobs */
  205    #include <wolfssl/wolfcrypt/port/caam/wolfcaam.h>
  206#endif
  207
  208#ifdef DEBUG_AESNI
  209    #include <stdio.h>
  210#endif
  211
  212#ifdef _MSC_VER
  213    /* 4127 warning constant while(1)  */
  214    #pragma warning(disable: 4127)
  215#endif
  216
  217#if (!defined(WOLFSSL_ARMASM) && FIPS_VERSION3_GE(6,0,0)) || \
  218    FIPS_VERSION3_GE(7,0,0)
  219    const unsigned int wolfCrypt_FIPS_aes_ro_sanity[2] =
  220                                                     { 0x1a2b3c4d, 0x00000002 };
  221    int wolfCrypt_FIPS_AES_sanity(void)
  222    {
  223        return 0;
  224    }
  225#endif
  226
  227/* Define AES implementation includes and functions */
  228#if defined(STM32_CRYPTO)
  229     /* STM32F2/F4/F7/L4/L5/H7/WB55 hardware AES support for ECB, CBC, CTR and GCM modes */
  230
  231#if defined(WOLFSSL_AES_DIRECT) || defined(HAVE_AESGCM) || defined(HAVE_AESCCM)
  232
  233    static WARN_UNUSED_RESULT int wc_AesEncrypt(
  234        Aes* aes, const byte* inBlock, byte* outBlock)
  235    {
  236        int ret = 0;
  237    #ifdef WOLFSSL_STM32_CUBEMX
  238        CRYP_HandleTypeDef hcryp;
  239    #else
  240        CRYP_InitTypeDef cryptInit;
  241        CRYP_KeyInitTypeDef keyInit;
  242    #endif
  243
  244#ifdef WC_DEBUG_CIPHER_LIFECYCLE
  245        ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
  246        if (ret < 0)
  247            return ret;
  248#endif
  249
  250    #ifdef WOLFSSL_STM32U5_DHUK
  251        ret = wolfSSL_CryptHwMutexLock();
  252        if (ret != 0)
  253            return ret;
  254
  255        /* Handle making use of wrapped key */
  256        if (aes->devId == WOLFSSL_STM32U5_DHUK_WRAPPED_DEVID) {
  257            CRYP_ConfigTypeDef Config = {0};
  258
  259            ret = wc_Stm32_Aes_UnWrap(aes, &hcryp, (const byte*)aes->key,
  260                aes->keylen, aes->dhukIV, aes->dhukIVLen);
  261            if (ret != HAL_OK) {
  262                WOLFSSL_MSG("Error with DHUK key unwrap");
  263                ret = BAD_FUNC_ARG;
  264            }
  265            /* reconfigure for using unwrapped key now */
  266            HAL_CRYP_GetConfig(&hcryp, &Config);
  267            Config.KeyMode   = CRYP_KEYMODE_NORMAL;
  268            Config.KeySelect = CRYP_KEYSEL_NORMAL;
  269            Config.Algorithm = CRYP_AES_ECB;
  270            Config.DataType  = CRYP_DATATYPE_8B;
  271            Config.DataWidthUnit = CRYP_DATAWIDTHUNIT_BYTE;
  272            HAL_CRYP_SetConfig(&hcryp, &Config);
  273        }
  274        else {
  275            ret = wc_Stm32_Aes_Init(aes, &hcryp, 1);
  276            if (ret == 0) {
  277                hcryp.Init.Algorithm  = CRYP_AES_ECB;
  278                ret = HAL_CRYP_Init(&hcryp);
  279                if (ret != HAL_OK) {
  280                    ret = BAD_FUNC_ARG;
  281                }
  282            }
  283        }
  284
  285        if (ret == HAL_OK) {
  286            ret = HAL_CRYP_Encrypt(&hcryp, (uint32_t*)inBlock, WC_AES_BLOCK_SIZE,
  287                    (uint32_t*)outBlock, STM32_HAL_TIMEOUT);
  288            if (ret != HAL_OK) {
  289                ret = WC_TIMEOUT_E;
  290            }
  291        }
  292        HAL_CRYP_DeInit(&hcryp);
  293    #elif defined(WOLFSSL_STM32_CUBEMX)
  294        ret = wc_Stm32_Aes_Init(aes, &hcryp, 0);
  295        if (ret != 0)
  296            return ret;
  297
  298        ret = wolfSSL_CryptHwMutexLock();
  299        if (ret != 0)
  300            return ret;
  301
  302    #if defined(STM32_HAL_V2)
  303        hcryp.Init.Algorithm  = CRYP_AES_ECB;
  304    #elif defined(STM32_CRYPTO_AES_ONLY)
  305        hcryp.Init.OperatingMode = CRYP_ALGOMODE_ENCRYPT;
  306        hcryp.Init.ChainingMode  = CRYP_CHAINMODE_AES_ECB;
  307        hcryp.Init.KeyWriteFlag  = CRYP_KEY_WRITE_ENABLE;
  308    #endif
  309        if (HAL_CRYP_Init(&hcryp) != HAL_OK) {
  310            ret = BAD_FUNC_ARG;
  311        }
  312
  313        if (ret == 0) {
  314        #if defined(STM32_HAL_V2)
  315            ret = HAL_CRYP_Encrypt(&hcryp, (uint32_t*)inBlock, WC_AES_BLOCK_SIZE,
  316                (uint32_t*)outBlock, STM32_HAL_TIMEOUT);
  317        #elif defined(STM32_CRYPTO_AES_ONLY)
  318            ret = HAL_CRYPEx_AES(&hcryp, (uint8_t*)inBlock, WC_AES_BLOCK_SIZE,
  319                outBlock, STM32_HAL_TIMEOUT);
  320        #else
  321            ret = HAL_CRYP_AESECB_Encrypt(&hcryp, (uint8_t*)inBlock, WC_AES_BLOCK_SIZE,
  322                outBlock, STM32_HAL_TIMEOUT);
  323        #endif
  324            if (ret != HAL_OK) {
  325                ret = WC_TIMEOUT_E;
  326            }
  327            HAL_CRYP_DeInit(&hcryp);
  328        }
  329
  330    #else /* Standard Peripheral Library */
  331        ret = wc_Stm32_Aes_Init(aes, &cryptInit, &keyInit);
  332        if (ret != 0)
  333            return ret;
  334
  335        ret = wolfSSL_CryptHwMutexLock();
  336        if (ret != 0)
  337            return ret;
  338
  339        /* reset registers to their default values */
  340        CRYP_DeInit();
  341
  342        /* setup key */
  343        CRYP_KeyInit(&keyInit);
  344
  345        /* set direction and mode */
  346        cryptInit.CRYP_AlgoDir  = CRYP_AlgoDir_Encrypt;
  347        cryptInit.CRYP_AlgoMode = CRYP_AlgoMode_AES_ECB;
  348        CRYP_Init(&cryptInit);
  349
  350        /* enable crypto processor */
  351        CRYP_Cmd(ENABLE);
  352
  353        /* flush IN/OUT FIFOs */
  354        CRYP_FIFOFlush();
  355
  356        CRYP_DataIn(*(uint32_t*)&inBlock[0]);
  357        CRYP_DataIn(*(uint32_t*)&inBlock[4]);
  358        CRYP_DataIn(*(uint32_t*)&inBlock[8]);
  359        CRYP_DataIn(*(uint32_t*)&inBlock[12]);
  360
  361        /* wait until the complete message has been processed */
  362        while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
  363
  364        *(uint32_t*)&outBlock[0]  = CRYP_DataOut();
  365        *(uint32_t*)&outBlock[4]  = CRYP_DataOut();
  366        *(uint32_t*)&outBlock[8]  = CRYP_DataOut();
  367        *(uint32_t*)&outBlock[12] = CRYP_DataOut();
  368
  369        /* disable crypto processor */
  370        CRYP_Cmd(DISABLE);
  371    #endif /* WOLFSSL_STM32_CUBEMX */
  372        wolfSSL_CryptHwMutexUnLock();
  373        wc_Stm32_Aes_Cleanup();
  374
  375        return ret;
  376    }
  377#endif /* WOLFSSL_AES_DIRECT || HAVE_AESGCM || HAVE_AESCCM */
  378
  379#ifdef HAVE_AES_DECRYPT
  380    #if defined(WOLFSSL_AES_DIRECT)
  381    static WARN_UNUSED_RESULT int wc_AesDecrypt(
  382        Aes* aes, const byte* inBlock, byte* outBlock)
  383    {
  384        int ret = 0;
  385    #ifdef WOLFSSL_STM32_CUBEMX
  386        CRYP_HandleTypeDef hcryp;
  387    #else
  388        CRYP_InitTypeDef cryptInit;
  389        CRYP_KeyInitTypeDef keyInit;
  390    #endif
  391
  392#ifdef WC_DEBUG_CIPHER_LIFECYCLE
  393        ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
  394        if (ret < 0)
  395            return ret;
  396#endif
  397
  398    #ifdef WOLFSSL_STM32U5_DHUK
  399        ret = wolfSSL_CryptHwMutexLock();
  400        if (ret != 0)
  401            return ret;
  402
  403        /* Handle making use of wrapped key */
  404        if (aes->devId == WOLFSSL_STM32U5_DHUK_WRAPPED_DEVID) {
  405            CRYP_ConfigTypeDef Config;
  406
  407            XMEMSET(&Config, 0, sizeof(Config));
  408            ret = wc_Stm32_Aes_UnWrap(aes, &hcryp, (const byte*)aes->key,
  409                aes->keylen, aes->dhukIV, aes->dhukIVLen);
  410            if (ret != HAL_OK) {
  411                WOLFSSL_MSG("Error with DHUK unwrap");
  412                ret = BAD_FUNC_ARG;
  413            }
  414            /* reconfigure for using unwrapped key now */
  415            HAL_CRYP_GetConfig(&hcryp, &Config);
  416            Config.KeyMode   = CRYP_KEYMODE_NORMAL;
  417            Config.KeySelect = CRYP_KEYSEL_NORMAL;
  418            Config.Algorithm = CRYP_AES_ECB;
  419            Config.DataType  = CRYP_DATATYPE_8B;
  420            Config.DataWidthUnit = CRYP_DATAWIDTHUNIT_BYTE;
  421            HAL_CRYP_SetConfig(&hcryp, &Config);
  422        }
  423        else {
  424            ret = wc_Stm32_Aes_Init(aes, &hcryp, 1);
  425            if (ret == 0) {
  426                hcryp.Init.Algorithm  = CRYP_AES_ECB;
  427                ret = HAL_CRYP_Init(&hcryp);
  428                if (ret != HAL_OK) {
  429                    ret = BAD_FUNC_ARG;
  430                }
  431            }
  432        }
  433
  434        if (ret == HAL_OK) {
  435            ret = HAL_CRYP_Decrypt(&hcryp, (uint32_t*)inBlock, WC_AES_BLOCK_SIZE,
  436                    (uint32_t*)outBlock, STM32_HAL_TIMEOUT);
  437            if (ret != HAL_OK) {
  438                ret = WC_TIMEOUT_E;
  439            }
  440        }
  441        HAL_CRYP_DeInit(&hcryp);
  442    #elif defined(WOLFSSL_STM32_CUBEMX)
  443        ret = wc_Stm32_Aes_Init(aes, &hcryp, 0);
  444        if (ret != 0)
  445            return ret;
  446
  447        ret = wolfSSL_CryptHwMutexLock();
  448        if (ret != 0)
  449            return ret;
  450
  451    #if defined(STM32_HAL_V2)
  452        hcryp.Init.Algorithm  = CRYP_AES_ECB;
  453    #elif defined(STM32_CRYPTO_AES_ONLY)
  454        hcryp.Init.OperatingMode = CRYP_ALGOMODE_KEYDERIVATION_DECRYPT;
  455        hcryp.Init.ChainingMode  = CRYP_CHAINMODE_AES_ECB;
  456        hcryp.Init.KeyWriteFlag  = CRYP_KEY_WRITE_ENABLE;
  457    #endif
  458        HAL_CRYP_Init(&hcryp);
  459
  460    #if defined(STM32_HAL_V2)
  461        ret = HAL_CRYP_Decrypt(&hcryp, (uint32_t*)inBlock, WC_AES_BLOCK_SIZE,
  462            (uint32_t*)outBlock, STM32_HAL_TIMEOUT);
  463    #elif defined(STM32_CRYPTO_AES_ONLY)
  464        ret = HAL_CRYPEx_AES(&hcryp, (uint8_t*)inBlock, WC_AES_BLOCK_SIZE,
  465            outBlock, STM32_HAL_TIMEOUT);
  466    #else
  467        ret = HAL_CRYP_AESECB_Decrypt(&hcryp, (uint8_t*)inBlock, WC_AES_BLOCK_SIZE,
  468            outBlock, STM32_HAL_TIMEOUT);
  469    #endif
  470        if (ret != HAL_OK) {
  471            ret = WC_TIMEOUT_E;
  472        }
  473        HAL_CRYP_DeInit(&hcryp);
  474
  475    #else /* Standard Peripheral Library */
  476        ret = wc_Stm32_Aes_Init(aes, &cryptInit, &keyInit);
  477        if (ret != 0)
  478            return ret;
  479
  480        ret = wolfSSL_CryptHwMutexLock();
  481        if (ret != 0)
  482            return ret;
  483
  484        /* reset registers to their default values */
  485        CRYP_DeInit();
  486
  487        /* set direction and key */
  488        CRYP_KeyInit(&keyInit);
  489        cryptInit.CRYP_AlgoDir  = CRYP_AlgoDir_Decrypt;
  490        cryptInit.CRYP_AlgoMode = CRYP_AlgoMode_AES_Key;
  491        CRYP_Init(&cryptInit);
  492
  493        /* enable crypto processor */
  494        CRYP_Cmd(ENABLE);
  495
  496        /* wait until decrypt key has been initialized */
  497        while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
  498
  499        /* set direction and mode */
  500        cryptInit.CRYP_AlgoDir  = CRYP_AlgoDir_Decrypt;
  501        cryptInit.CRYP_AlgoMode = CRYP_AlgoMode_AES_ECB;
  502        CRYP_Init(&cryptInit);
  503
  504        /* enable crypto processor */
  505        CRYP_Cmd(ENABLE);
  506
  507        /* flush IN/OUT FIFOs */
  508        CRYP_FIFOFlush();
  509
  510        CRYP_DataIn(*(uint32_t*)&inBlock[0]);
  511        CRYP_DataIn(*(uint32_t*)&inBlock[4]);
  512        CRYP_DataIn(*(uint32_t*)&inBlock[8]);
  513        CRYP_DataIn(*(uint32_t*)&inBlock[12]);
  514
  515        /* wait until the complete message has been processed */
  516        while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
  517
  518        *(uint32_t*)&outBlock[0]  = CRYP_DataOut();
  519        *(uint32_t*)&outBlock[4]  = CRYP_DataOut();
  520        *(uint32_t*)&outBlock[8]  = CRYP_DataOut();
  521        *(uint32_t*)&outBlock[12] = CRYP_DataOut();
  522
  523        /* disable crypto processor */
  524        CRYP_Cmd(DISABLE);
  525    #endif /* WOLFSSL_STM32_CUBEMX */
  526        wolfSSL_CryptHwMutexUnLock();
  527        wc_Stm32_Aes_Cleanup();
  528
  529        return ret;
  530    }
  531    #endif /* WOLFSSL_AES_DIRECT */
  532#endif /* HAVE_AES_DECRYPT */
  533
  534#elif defined(HAVE_COLDFIRE_SEC)
  535    /* Freescale Coldfire SEC support for CBC mode.
  536     * NOTE: no support for AES-CTR/GCM/CCM/Direct */
  537    #include "sec.h"
  538    #include "mcf5475_sec.h"
  539    #include "mcf5475_siu.h"
  540#elif defined(FREESCALE_LTC)
  541    #include "fsl_ltc.h"
  542    #if defined(FREESCALE_LTC_AES_GCM)
  543        #undef NEED_AES_TABLES
  544        #undef GCM_TABLE
  545    #endif
  546
  547        /* if LTC doesn't have GCM, use software with LTC AES ECB mode */
  548        static WARN_UNUSED_RESULT int wc_AesEncrypt(
  549            Aes* aes, const byte* inBlock, byte* outBlock)
  550        {
  551            word32 keySize = 0;
  552            byte* key = (byte*)aes->key;
  553            int ret = wc_AesGetKeySize(aes, &keySize);
  554            if (ret != 0)
  555                return ret;
  556
  557#ifdef WC_DEBUG_CIPHER_LIFECYCLE
  558            ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
  559            if (ret < 0)
  560                return ret;
  561#endif
  562
  563            if (wolfSSL_CryptHwMutexLock() == 0) {
  564                LTC_AES_EncryptEcb(LTC_BASE, inBlock, outBlock, WC_AES_BLOCK_SIZE,
  565                    key, keySize);
  566                wolfSSL_CryptHwMutexUnLock();
  567            }
  568            return 0;
  569        }
  570        #ifdef HAVE_AES_DECRYPT
  571        static WARN_UNUSED_RESULT int wc_AesDecrypt(
  572            Aes* aes, const byte* inBlock, byte* outBlock)
  573        {
  574            word32 keySize = 0;
  575            byte* key = (byte*)aes->key;
  576            int ret = wc_AesGetKeySize(aes, &keySize);
  577            if (ret != 0)
  578                return ret;
  579
  580#ifdef WC_DEBUG_CIPHER_LIFECYCLE
  581            ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
  582            if (ret < 0)
  583                return ret;
  584#endif
  585
  586            if (wolfSSL_CryptHwMutexLock() == 0) {
  587                LTC_AES_DecryptEcb(LTC_BASE, inBlock, outBlock, WC_AES_BLOCK_SIZE,
  588                    key, keySize, kLTC_EncryptKey);
  589                wolfSSL_CryptHwMutexUnLock();
  590            }
  591            return 0;
  592        }
  593        #endif
  594
  595#elif defined(WOLFSSL_PIC32MZ_CRYPT)
  596
  597    #include <wolfssl/wolfcrypt/port/pic32/pic32mz-crypt.h>
  598
  599    #if defined(HAVE_AESGCM) || defined(WOLFSSL_AES_DIRECT)
  600    static WARN_UNUSED_RESULT int wc_AesEncrypt(
  601        Aes* aes, const byte* inBlock, byte* outBlock)
  602    {
  603#ifdef WC_DEBUG_CIPHER_LIFECYCLE
  604        {
  605            int ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
  606            if (ret < 0)
  607                return ret;
  608        }
  609#endif
  610        /* Thread mutex protection handled in Pic32Crypto */
  611        return wc_Pic32AesCrypt(aes->key, aes->keylen, NULL, 0,
  612            outBlock, inBlock, WC_AES_BLOCK_SIZE,
  613            PIC32_ENCRYPTION, PIC32_ALGO_AES, PIC32_CRYPTOALGO_RECB);
  614    }
  615    #endif
  616
  617    #if defined(HAVE_AES_DECRYPT) && defined(WOLFSSL_AES_DIRECT)
  618    static WARN_UNUSED_RESULT int wc_AesDecrypt(
  619        Aes* aes, const byte* inBlock, byte* outBlock)
  620    {
  621#ifdef WC_DEBUG_CIPHER_LIFECYCLE
  622        {
  623            int ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
  624            if (ret < 0)
  625                return ret;
  626        }
  627#endif
  628        /* Thread mutex protection handled in Pic32Crypto */
  629        return wc_Pic32AesCrypt(aes->key, aes->keylen, NULL, 0,
  630            outBlock, inBlock, WC_AES_BLOCK_SIZE,
  631            PIC32_DECRYPTION, PIC32_ALGO_AES, PIC32_CRYPTOALGO_RECB);
  632    }
  633    #endif
  634
  635#elif defined(WOLFSSL_NRF51_AES)
  636    /* Use built-in AES hardware - AES 128 ECB Encrypt Only */
  637    #include "wolfssl/wolfcrypt/port/nrf51.h"
  638
  639    static WARN_UNUSED_RESULT int wc_AesEncrypt(
  640        Aes* aes, const byte* inBlock, byte* outBlock)
  641    {
  642        int ret;
  643
  644#ifdef WC_DEBUG_CIPHER_LIFECYCLE
  645        ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
  646        if (ret < 0)
  647            return ret;
  648#endif
  649
  650        ret = wolfSSL_CryptHwMutexLock();
  651        if (ret == 0) {
  652            ret = nrf51_aes_encrypt(inBlock, (byte*)aes->key, aes->rounds,
  653                                    outBlock);
  654            wolfSSL_CryptHwMutexUnLock();
  655        }
  656        return ret;
  657    }
  658
  659    #ifdef HAVE_AES_DECRYPT
  660        #error nRF51 AES Hardware does not support decrypt
  661    #endif /* HAVE_AES_DECRYPT */
  662
  663#elif defined(WOLFSSL_ESP32_CRYPT) && \
  664     !defined(NO_WOLFSSL_ESP32_CRYPT_AES)
  665    #include <esp_log.h>
  666    #include <wolfssl/wolfcrypt/port/Espressif/esp32-crypt.h>
  667    #define TAG "aes"
  668
  669    /* We'll use SW for fallback:
  670     *   unsupported key lengths. (e.g. ESP32-S3)
  671     *   chipsets not implemented.
  672     *   hardware busy. */
  673    #define NEED_AES_TABLES
  674    #define NEED_AES_HW_FALLBACK
  675    #define NEED_SOFTWARE_AES_SETKEY
  676    #undef  WOLFSSL_AES_DIRECT
  677    #define WOLFSSL_AES_DIRECT
  678
  679    /* Encrypt: If we choose to never have a fallback to SW: */
  680    #if !defined(NEED_AES_HW_FALLBACK) && \
  681        (defined(HAVE_AESGCM) || defined(WOLFSSL_AES_DIRECT))
  682    /* calling this one when NO_AES_192 is defined */
  683    static WARN_UNUSED_RESULT int wc_AesEncrypt(
  684        Aes* aes, const byte* inBlock, byte* outBlock)
  685    {
  686        int ret;
  687
  688    #ifdef WC_DEBUG_CIPHER_LIFECYCLE
  689        ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
  690        if (ret < 0)
  691            return ret;
  692    #endif
  693
  694        /* Thread mutex protection handled in esp_aes_hw_InUse */
  695    #ifdef NEED_AES_HW_FALLBACK
  696        if (wc_esp32AesSupportedKeyLen(aes)) {
  697            ret = wc_esp32AesEncrypt(aes, inBlock, outBlock);
  698        }
  699    #else
  700        ret = wc_esp32AesEncrypt(aes, inBlock, outBlock);
  701    #endif
  702        return ret;
  703    }
  704    #endif
  705
  706    /* Decrypt: If we choose to never have a fallback to SW: */
  707    #if !defined(NEED_AES_HW_FALLBACK) && \
  708        (defined(HAVE_AES_DECRYPT) && defined(WOLFSSL_AES_DIRECT))
  709    static WARN_UNUSED_RESULT int wc_AesDecrypt(
  710        Aes* aes, const byte* inBlock, byte* outBlock)
  711    {
  712        int ret = 0;
  713#ifdef WC_DEBUG_CIPHER_LIFECYCLE
  714        ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
  715        if (ret < 0)
  716            return ret;
  717#endif
  718        /* Thread mutex protection handled in esp_aes_hw_InUse */
  719    #ifdef NEED_AES_HW_FALLBACK
  720        if (wc_esp32AesSupportedKeyLen(aes)) {
  721            ret = wc_esp32AesDecrypt(aes, inBlock, outBlock);
  722        }
  723        else {
  724            ret = wc_AesDecrypt_SW(aes, inBlock, outBlock);
  725        }
  726    #else
  727        /* if we don't need fallback, always use HW */
  728        ret = wc_esp32AesDecrypt(aes, inBlock, outBlock);
  729    #endif
  730        return ret;
  731    }
  732    #endif
  733
  734#elif defined(WOLFSSL_AESNI)
  735
  736    #define NEED_AES_TABLES
  737
  738    /* Each platform needs to query info type 1 from cpuid to see if aesni is
  739     * supported. Also, let's setup a macro for proper linkage w/o ABI conflicts
  740     */
  741
  742    #ifndef AESNI_ALIGN
  743        #define AESNI_ALIGN 16
  744    #endif
  745
  746    /* note that all write access to these static variables must be idempotent,
  747     * as arranged by Check_CPU_support_AES(), else they will be susceptible to
  748     * data races.
  749     */
  750    static int checkedAESNI = 0;
  751    static int haveAESNI  = 0;
  752    static cpuid_flags_t intel_flags = WC_CPUID_INITIALIZER;
  753
  754    static WARN_UNUSED_RESULT int Check_CPU_support_AES(void)
  755    {
  756        cpuid_get_flags_ex(&intel_flags);
  757
  758        return IS_INTEL_AESNI(intel_flags) != 0;
  759    }
  760
  761
  762    /* tell C compiler these are asm functions in case any mix up of ABI underscore
  763       prefix between clang/gcc/llvm etc */
  764    #ifdef HAVE_AES_CBC
  765        void AES_CBC_encrypt_AESNI(const unsigned char* in, unsigned char* out,
  766                             unsigned char* ivec, unsigned long length,
  767                             const unsigned char* KS, int nr)
  768                             XASM_LINK("AES_CBC_encrypt_AESNI");
  769
  770        #ifdef HAVE_AES_DECRYPT
  771            #if defined(WOLFSSL_AESNI_BY4) || defined(WOLFSSL_X86_BUILD)
  772                void AES_CBC_decrypt_AESNI_by4(const unsigned char* in, unsigned char* out,
  773                                         unsigned char* ivec, unsigned long length,
  774                                         const unsigned char* KS, int nr)
  775                                         XASM_LINK("AES_CBC_decrypt_AESNI_by4");
  776            #elif defined(WOLFSSL_AESNI_BY6)
  777                void AES_CBC_decrypt_AESNI_by6(const unsigned char* in, unsigned char* out,
  778                                         unsigned char* ivec, unsigned long length,
  779                                         const unsigned char* KS, int nr)
  780                                         XASM_LINK("AES_CBC_decrypt_AESNI_by6");
  781            #else /* WOLFSSL_AESNI_BYx */
  782                void AES_CBC_decrypt_AESNI_by8(const unsigned char* in, unsigned char* out,
  783                                         unsigned char* ivec, unsigned long length,
  784                                         const unsigned char* KS, int nr)
  785                                         XASM_LINK("AES_CBC_decrypt_AESNI_by8");
  786            #endif /* WOLFSSL_AESNI_BYx */
  787        #endif /* HAVE_AES_DECRYPT */
  788    #endif /* HAVE_AES_CBC */
  789
  790    void AES_ECB_encrypt_AESNI(const unsigned char* in, unsigned char* out,
  791                         unsigned long length, const unsigned char* KS, int nr)
  792                         XASM_LINK("AES_ECB_encrypt_AESNI");
  793
  794    #ifdef HAVE_AES_DECRYPT
  795        void AES_ECB_decrypt_AESNI(const unsigned char* in, unsigned char* out,
  796                             unsigned long length, const unsigned char* KS, int nr)
  797                             XASM_LINK("AES_ECB_decrypt_AESNI");
  798    #endif
  799
  800    void AES_128_Key_Expansion_AESNI(const unsigned char* userkey,
  801                               unsigned char* key_schedule)
  802                               XASM_LINK("AES_128_Key_Expansion_AESNI");
  803
  804    void AES_192_Key_Expansion_AESNI(const unsigned char* userkey,
  805                               unsigned char* key_schedule)
  806                               XASM_LINK("AES_192_Key_Expansion_AESNI");
  807
  808    void AES_256_Key_Expansion_AESNI(const unsigned char* userkey,
  809                               unsigned char* key_schedule)
  810                               XASM_LINK("AES_256_Key_Expansion_AESNI");
  811
  812
  813    static WARN_UNUSED_RESULT int AES_set_encrypt_key_AESNI(
  814        const unsigned char *userKey, const int bits, Aes* aes)
  815    {
  816        int ret;
  817
  818        ASSERT_SAVED_VECTOR_REGISTERS();
  819
  820        if (!userKey || !aes)
  821            return BAD_FUNC_ARG;
  822
  823        switch (bits) {
  824            case 128:
  825               AES_128_Key_Expansion_AESNI (userKey,(byte*)aes->key); aes->rounds = 10;
  826               return 0;
  827            case 192:
  828               AES_192_Key_Expansion_AESNI (userKey,(byte*)aes->key); aes->rounds = 12;
  829               return 0;
  830            case 256:
  831               AES_256_Key_Expansion_AESNI (userKey,(byte*)aes->key); aes->rounds = 14;
  832               return 0;
  833            default:
  834                ret = BAD_FUNC_ARG;
  835        }
  836
  837        return ret;
  838    }
  839
  840    #ifdef HAVE_AES_DECRYPT
  841        static WARN_UNUSED_RESULT int AES_set_decrypt_key_AESNI(
  842            const unsigned char* userKey, const int bits, Aes* aes)
  843        {
  844            word32 nr;
  845            WC_DECLARE_VAR(temp_key, Aes, 1, 0);
  846            __m128i *Key_Schedule;
  847            __m128i *Temp_Key_Schedule;
  848
  849            ASSERT_SAVED_VECTOR_REGISTERS();
  850
  851            if (!userKey || !aes)
  852                return BAD_FUNC_ARG;
  853
  854#ifdef WOLFSSL_SMALL_STACK
  855            if ((temp_key = (Aes *)XMALLOC(sizeof *aes, aes->heap,
  856                                           DYNAMIC_TYPE_AES)) == NULL)
  857                return MEMORY_E;
  858#endif
  859
  860            if (AES_set_encrypt_key_AESNI(userKey,bits,temp_key)
  861                == WC_NO_ERR_TRACE(BAD_FUNC_ARG)) {
  862                WC_FREE_VAR_EX(temp_key, aes->heap, DYNAMIC_TYPE_AES);
  863                return BAD_FUNC_ARG;
  864            }
  865
  866            Key_Schedule = (__m128i*)aes->key;
  867            Temp_Key_Schedule = (__m128i*)temp_key->key;
  868
  869            nr = temp_key->rounds;
  870            aes->rounds = nr;
  871
  872            Key_Schedule[nr] = Temp_Key_Schedule[0];
  873            Key_Schedule[nr-1] = _mm_aesimc_si128(Temp_Key_Schedule[1]);
  874            Key_Schedule[nr-2] = _mm_aesimc_si128(Temp_Key_Schedule[2]);
  875            Key_Schedule[nr-3] = _mm_aesimc_si128(Temp_Key_Schedule[3]);
  876            Key_Schedule[nr-4] = _mm_aesimc_si128(Temp_Key_Schedule[4]);
  877            Key_Schedule[nr-5] = _mm_aesimc_si128(Temp_Key_Schedule[5]);
  878            Key_Schedule[nr-6] = _mm_aesimc_si128(Temp_Key_Schedule[6]);
  879            Key_Schedule[nr-7] = _mm_aesimc_si128(Temp_Key_Schedule[7]);
  880            Key_Schedule[nr-8] = _mm_aesimc_si128(Temp_Key_Schedule[8]);
  881            Key_Schedule[nr-9] = _mm_aesimc_si128(Temp_Key_Schedule[9]);
  882
  883            if (nr>10) {
  884                Key_Schedule[nr-10] = _mm_aesimc_si128(Temp_Key_Schedule[10]);
  885                Key_Schedule[nr-11] = _mm_aesimc_si128(Temp_Key_Schedule[11]);
  886            }
  887
  888            if (nr>12) {
  889                Key_Schedule[nr-12] = _mm_aesimc_si128(Temp_Key_Schedule[12]);
  890                Key_Schedule[nr-13] = _mm_aesimc_si128(Temp_Key_Schedule[13]);
  891            }
  892
  893            Key_Schedule[0] = Temp_Key_Schedule[nr];
  894
  895            WC_FREE_VAR_EX(temp_key, aes->heap, DYNAMIC_TYPE_AES);
  896
  897            return 0;
  898        }
  899    #endif /* HAVE_AES_DECRYPT */
  900
  901#elif defined(WOLFSSL_ARMASM)
  902#if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
  903static cpuid_flags_t cpuid_flags = WC_CPUID_INITIALIZER;
  904
  905static void Check_CPU_support_HwCrypto(Aes* aes)
  906{
  907    cpuid_get_flags_ex(&cpuid_flags);
  908    aes->use_aes_hw_crypto = IS_AARCH64_AES(cpuid_flags);
  909#ifdef HAVE_AESGCM
  910    aes->use_pmull_hw_crypto = IS_AARCH64_PMULL(cpuid_flags);
  911    aes->use_sha3_hw_crypto = IS_AARCH64_SHA3(cpuid_flags);
  912#endif
  913}
  914#endif /* __aarch64__ && !WOLFSSL_ARMASM_NO_HW_CRYPTO */
  915
  916#if defined(WOLFSSL_AES_DIRECT) || defined(HAVE_AESCCM) || \
  917    defined(WOLFSSL_AESGCM_STREAM)
  918static WARN_UNUSED_RESULT int wc_AesEncrypt(Aes* aes, const byte* inBlock,
  919    byte* outBlock)
  920{
  921#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
  922#if !defined(__aarch64__)
  923    AES_encrypt_AARCH32(inBlock, outBlock, (byte*)aes->key, (int)aes->rounds);
  924#else
  925    if (aes->use_aes_hw_crypto) {
  926        AES_encrypt_AARCH64(inBlock, outBlock, (byte*)aes->key,
  927           (int)aes->rounds);
  928    }
  929    else
  930#endif /* !__aarch64__ */
  931#endif /* !WOLFSSL_ARMASM_NO_HW_CRYPTO */
  932#if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON) && \
  933    defined(WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP)
  934    {
  935        AES_ECB_encrypt_NEON(inBlock, outBlock, WC_AES_BLOCK_SIZE,
  936            (const unsigned char*)aes->key, aes->rounds);
  937    }
  938#elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
  939    {
  940        AES_ECB_encrypt(inBlock, outBlock, WC_AES_BLOCK_SIZE, (byte*)aes->key,
  941            (int)aes->rounds);
  942    }
  943#endif
  944
  945    return 0;
  946}
  947#endif
  948
  949#if defined(HAVE_AES_DECRYPT) && defined(WOLFSSL_AES_DIRECT)
  950static WARN_UNUSED_RESULT int wc_AesDecrypt(Aes* aes, const byte* inBlock,
  951    byte* outBlock)
  952{
  953#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
  954#if !defined(__aarch64__)
  955    AES_decrypt_AARCH32(inBlock, outBlock, (byte*)aes->key, (int)aes->rounds);
  956#else
  957    if (aes->use_aes_hw_crypto) {
  958        AES_decrypt_AARCH64(inBlock, outBlock, (byte*)aes->key,
  959            (int)aes->rounds);
  960    }
  961    else
  962#endif /* !__aarch64__ */
  963#endif /* !WOLFSSL_ARMASM_NO_HW_CRYPTO */
  964#if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON) && \
  965    defined(WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP)
  966    {
  967        AES_ECB_decrypt_NEON(inBlock, outBlock, WC_AES_BLOCK_SIZE,
  968            (byte*)aes->key, (int)aes->rounds);
  969    }
  970#elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
  971    {
  972        AES_ECB_decrypt(inBlock, outBlock, WC_AES_BLOCK_SIZE, (byte*)aes->key,
  973            (int)aes->rounds);
  974    }
  975#endif
  976    return 0;
  977}
  978#endif /* HAVE_AES_DECRYPT && WOLFSSL_AES_DIRECT */
  979
  980#elif defined(FREESCALE_MMCAU)
  981    /* Freescale mmCAU hardware AES support for Direct, CBC, CCM, GCM modes
  982     * through the CAU/mmCAU library. Documentation located in
  983     * ColdFire/ColdFire+ CAU and Kinetis mmCAU Software Library User
  984     * Guide (See note in README). */
  985    #ifdef FREESCALE_MMCAU_CLASSIC
  986        /* MMCAU 1.4 library used with non-KSDK / classic MQX builds */
  987        #include "cau_api.h"
  988    #else
  989        #include "fsl_mmcau.h"
  990    #endif
  991
  992    static WARN_UNUSED_RESULT int wc_AesEncrypt(
  993        Aes* aes, const byte* inBlock, byte* outBlock)
  994    {
  995#ifdef WC_DEBUG_CIPHER_LIFECYCLE
  996        {
  997            int ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
  998            if (ret < 0)
  999                return ret;
 1000        }
 1001#endif
 1002
 1003        if (wolfSSL_CryptHwMutexLock() == 0) {
 1004        #ifdef FREESCALE_MMCAU_CLASSIC
 1005            if ((wc_ptr_t)outBlock % WOLFSSL_MMCAU_ALIGNMENT) {
 1006                WOLFSSL_MSG("Bad cau_aes_encrypt alignment");
 1007                return BAD_ALIGN_E;
 1008            }
 1009            cau_aes_encrypt(inBlock, (byte*)aes->key, aes->rounds, outBlock);
 1010        #else
 1011            MMCAU_AES_EncryptEcb(inBlock, (byte*)aes->key, aes->rounds,
 1012                                 outBlock);
 1013        #endif
 1014            wolfSSL_CryptHwMutexUnLock();
 1015        }
 1016        return 0;
 1017    }
 1018    #ifdef HAVE_AES_DECRYPT
 1019    static WARN_UNUSED_RESULT int wc_AesDecrypt(
 1020        Aes* aes, const byte* inBlock, byte* outBlock)
 1021    {
 1022#ifdef WC_DEBUG_CIPHER_LIFECYCLE
 1023        {
 1024            int ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
 1025            if (ret < 0)
 1026                return ret;
 1027        }
 1028#endif
 1029        if (wolfSSL_CryptHwMutexLock() == 0) {
 1030        #ifdef FREESCALE_MMCAU_CLASSIC
 1031            if ((wc_ptr_t)outBlock % WOLFSSL_MMCAU_ALIGNMENT) {
 1032                WOLFSSL_MSG("Bad cau_aes_decrypt alignment");
 1033                return BAD_ALIGN_E;
 1034            }
 1035            cau_aes_decrypt(inBlock, (byte*)aes->key, aes->rounds, outBlock);
 1036        #else
 1037            MMCAU_AES_DecryptEcb(inBlock, (byte*)aes->key, aes->rounds,
 1038                                 outBlock);
 1039        #endif
 1040            wolfSSL_CryptHwMutexUnLock();
 1041        }
 1042        return 0;
 1043    }
 1044    #endif /* HAVE_AES_DECRYPT */
 1045
 1046#elif (defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES) \
 1047        && !defined(WOLFSSL_QNX_CAAM)) || \
 1048      ((defined(WOLFSSL_AFALG) || defined(WOLFSSL_DEVCRYPTO_AES)) && \
 1049        defined(HAVE_AESCCM))
 1050        static WARN_UNUSED_RESULT int wc_AesEncrypt(
 1051            Aes* aes, const byte* inBlock, byte* outBlock)
 1052        {
 1053#ifdef WC_DEBUG_CIPHER_LIFECYCLE
 1054            {
 1055                int ret =
 1056                    wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
 1057                if (ret < 0)
 1058                    return ret;
 1059            }
 1060#endif
 1061            return wc_AesEncryptDirect(aes, outBlock, inBlock);
 1062        }
 1063
 1064#elif defined(WOLFSSL_AFALG)
 1065    /* implemented in wolfcrypt/src/port/af_alg/afalg_aes.c */
 1066
 1067#elif defined(WOLFSSL_DEVCRYPTO_AES)
 1068    /* implemented in wolfcrypt/src/port/devcrypto/devcrypto_aes.c */
 1069
 1070#elif defined(WOLFSSL_SCE) && !defined(WOLFSSL_SCE_NO_AES)
 1071    #include "hal_data.h"
 1072
 1073    #ifndef WOLFSSL_SCE_AES256_HANDLE
 1074        #define WOLFSSL_SCE_AES256_HANDLE g_sce_aes_256
 1075    #endif
 1076
 1077    #ifndef WOLFSSL_SCE_AES192_HANDLE
 1078        #define WOLFSSL_SCE_AES192_HANDLE g_sce_aes_192
 1079    #endif
 1080
 1081    #ifndef WOLFSSL_SCE_AES128_HANDLE
 1082        #define WOLFSSL_SCE_AES128_HANDLE g_sce_aes_128
 1083    #endif
 1084
 1085    static WARN_UNUSED_RESULT int AES_ECB_encrypt(
 1086        Aes* aes, const byte* inBlock, byte* outBlock, int sz)
 1087    {
 1088        word32 ret;
 1089
 1090        if (WOLFSSL_SCE_GSCE_HANDLE.p_cfg->endian_flag ==
 1091                CRYPTO_WORD_ENDIAN_BIG) {
 1092            ByteReverseWords((word32*)inBlock, (word32*)inBlock, sz);
 1093        }
 1094
 1095        switch (aes->keylen) {
 1096        #ifdef WOLFSSL_AES_128
 1097            case AES_128_KEY_SIZE:
 1098                ret = WOLFSSL_SCE_AES128_HANDLE.p_api->encrypt(
 1099                        WOLFSSL_SCE_AES128_HANDLE.p_ctrl, aes->key,
 1100                        NULL, (sz / sizeof(word32)), (word32*)inBlock,
 1101                        (word32*)outBlock);
 1102                break;
 1103        #endif
 1104        #ifdef WOLFSSL_AES_192
 1105            case AES_192_KEY_SIZE:
 1106                ret = WOLFSSL_SCE_AES192_HANDLE.p_api->encrypt(
 1107                        WOLFSSL_SCE_AES192_HANDLE.p_ctrl, aes->key,
 1108                        NULL, (sz / sizeof(word32)), (word32*)inBlock,
 1109                        (word32*)outBlock);
 1110                break;
 1111        #endif
 1112        #ifdef WOLFSSL_AES_256
 1113            case AES_256_KEY_SIZE:
 1114                ret = WOLFSSL_SCE_AES256_HANDLE.p_api->encrypt(
 1115                        WOLFSSL_SCE_AES256_HANDLE.p_ctrl, aes->key,
 1116                        NULL, (sz / sizeof(word32)), (word32*)inBlock,
 1117                        (word32*)outBlock);
 1118                break;
 1119        #endif
 1120            default:
 1121                WOLFSSL_MSG("Unknown key size");
 1122                return BAD_FUNC_ARG;
 1123        }
 1124
 1125        if (ret != SSP_SUCCESS) {
 1126            /* revert input */
 1127            ByteReverseWords((word32*)inBlock, (word32*)inBlock, sz);
 1128            return WC_HW_E;
 1129        }
 1130
 1131        if (WOLFSSL_SCE_GSCE_HANDLE.p_cfg->endian_flag ==
 1132                CRYPTO_WORD_ENDIAN_BIG) {
 1133            ByteReverseWords((word32*)outBlock, (word32*)outBlock, sz);
 1134            if (inBlock != outBlock) {
 1135                /* revert input */
 1136                ByteReverseWords((word32*)inBlock, (word32*)inBlock, sz);
 1137            }
 1138        }
 1139        return 0;
 1140    }
 1141
 1142    #if defined(HAVE_AES_DECRYPT)
 1143    static WARN_UNUSED_RESULT int AES_ECB_decrypt(
 1144        Aes* aes, const byte* inBlock, byte* outBlock, int sz)
 1145    {
 1146        word32 ret;
 1147
 1148        if (WOLFSSL_SCE_GSCE_HANDLE.p_cfg->endian_flag ==
 1149                CRYPTO_WORD_ENDIAN_BIG) {
 1150            ByteReverseWords((word32*)inBlock, (word32*)inBlock, sz);
 1151        }
 1152
 1153        switch (aes->keylen) {
 1154        #ifdef WOLFSSL_AES_128
 1155            case AES_128_KEY_SIZE:
 1156                ret = WOLFSSL_SCE_AES128_HANDLE.p_api->decrypt(
 1157                        WOLFSSL_SCE_AES128_HANDLE.p_ctrl, aes->key, aes->reg,
 1158                        (sz / sizeof(word32)), (word32*)inBlock,
 1159                        (word32*)outBlock);
 1160                break;
 1161        #endif
 1162        #ifdef WOLFSSL_AES_192
 1163            case AES_192_KEY_SIZE:
 1164                ret = WOLFSSL_SCE_AES192_HANDLE.p_api->decrypt(
 1165                        WOLFSSL_SCE_AES192_HANDLE.p_ctrl, aes->key, aes->reg,
 1166                        (sz / sizeof(word32)), (word32*)inBlock,
 1167                        (word32*)outBlock);
 1168                break;
 1169        #endif
 1170        #ifdef WOLFSSL_AES_256
 1171            case AES_256_KEY_SIZE:
 1172                ret = WOLFSSL_SCE_AES256_HANDLE.p_api->decrypt(
 1173                        WOLFSSL_SCE_AES256_HANDLE.p_ctrl, aes->key, aes->reg,
 1174                        (sz / sizeof(word32)), (word32*)inBlock,
 1175                        (word32*)outBlock);
 1176                break;
 1177        #endif
 1178            default:
 1179                WOLFSSL_MSG("Unknown key size");
 1180                return BAD_FUNC_ARG;
 1181        }
 1182        if (ret != SSP_SUCCESS) {
 1183            return WC_HW_E;
 1184        }
 1185
 1186        if (WOLFSSL_SCE_GSCE_HANDLE.p_cfg->endian_flag ==
 1187                CRYPTO_WORD_ENDIAN_BIG) {
 1188            ByteReverseWords((word32*)outBlock, (word32*)outBlock, sz);
 1189            if (inBlock != outBlock) {
 1190                /* revert input */
 1191                ByteReverseWords((word32*)inBlock, (word32*)inBlock, sz);
 1192            }
 1193        }
 1194
 1195        return 0;
 1196    }
 1197    #endif /* HAVE_AES_DECRYPT */
 1198
 1199    #if defined(HAVE_AESGCM) || defined(WOLFSSL_AES_DIRECT)
 1200    static WARN_UNUSED_RESULT int wc_AesEncrypt(
 1201        Aes* aes, const byte* inBlock, byte* outBlock)
 1202    {
 1203#ifdef WC_DEBUG_CIPHER_LIFECYCLE
 1204        {
 1205            int ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
 1206            if (ret < 0)
 1207                return ret;
 1208        }
 1209#endif
 1210        return AES_ECB_encrypt(aes, inBlock, outBlock, WC_AES_BLOCK_SIZE);
 1211    }
 1212    #endif
 1213
 1214    #if defined(HAVE_AES_DECRYPT) && defined(WOLFSSL_AES_DIRECT)
 1215    static WARN_UNUSED_RESULT int wc_AesDecrypt(
 1216        Aes* aes, const byte* inBlock, byte* outBlock)
 1217    {
 1218#ifdef WC_DEBUG_CIPHER_LIFECYCLE
 1219        {
 1220            int ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
 1221            if (ret < 0)
 1222                return ret;
 1223        }
 1224#endif
 1225        return AES_ECB_decrypt(aes, inBlock, outBlock, WC_AES_BLOCK_SIZE);
 1226    }
 1227    #endif
 1228
 1229#elif defined(WOLFSSL_KCAPI_AES)
 1230    /* Only CBC and GCM are in wolfcrypt/src/port/kcapi/kcapi_aes.c */
 1231    #if defined(WOLFSSL_AES_COUNTER) || defined(HAVE_AESCCM) || \
 1232        defined(WOLFSSL_CMAC) || defined(WOLFSSL_AES_OFB) || \
 1233        defined(WOLFSSL_AES_CFB) || defined(HAVE_AES_ECB) || \
 1234        defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_XTS) || \
 1235        (defined(HAVE_AES_CBC) && defined(WOLFSSL_NO_KCAPI_AES_CBC))
 1236
 1237        #define NEED_AES_TABLES
 1238    #endif
 1239#elif defined(WOLFSSL_HAVE_PSA) && !defined(WOLFSSL_PSA_NO_AES)
 1240/* implemented in wolfcrypt/src/port/psa/psa_aes.c */
 1241
 1242#elif defined(WOLFSSL_RISCV_ASM)
 1243/* implemented in wolfcrypt/src/port/riscv/riscv-64-aes.c */
 1244
 1245#elif defined(WOLFSSL_SILABS_SE_ACCEL)
 1246/* implemented in wolfcrypt/src/port/silabs/silabs_aes.c */
 1247
 1248#elif defined(WOLFSSL_PSOC6_CRYPTO)
 1249
 1250    #if (defined(HAVE_AESGCM) || defined(WOLFSSL_AES_DIRECT))
 1251        static WARN_UNUSED_RESULT int wc_AesEncrypt(
 1252            Aes* aes, const byte* inBlock, byte* outBlock)
 1253        {
 1254            return wc_Psoc6_Aes_Encrypt(aes, inBlock, outBlock);
 1255        }
 1256    #endif
 1257
 1258    #if defined(HAVE_AES_DECRYPT) && defined(WOLFSSL_AES_DIRECT)
 1259        static WARN_UNUSED_RESULT int wc_AesDecrypt(
 1260            Aes* aes, const byte* inBlock, byte* outBlock)
 1261        {
 1262            return wc_Psoc6_Aes_Decrypt(aes, inBlock, outBlock);
 1263        }
 1264
 1265    #endif
 1266#elif defined(WOLF_CRYPTO_CB_ONLY_AES)
 1267    /* No software implementation AES T-tables, S-box, Rcon and the C key
 1268     * schedule are stripped. */
 1269#else
 1270
 1271    /* using wolfCrypt software implementation */
 1272    #define NEED_AES_TABLES
 1273#endif
 1274
 1275
 1276
 1277#if defined(WC_AES_BITSLICED) && !defined(HAVE_AES_ECB)
 1278    #error "When WC_AES_BITSLICED is defined, HAVE_AES_ECB is needed."
 1279#endif
 1280
 1281#ifdef NEED_AES_TABLES
 1282
 1283#ifndef WC_AES_BITSLICED
 1284#if defined(__aarch64__) || !defined(WOLFSSL_ARMASM)
 1285#if !defined(WOLFSSL_ESP32_CRYPT) || \
 1286    (defined(NO_ESP32_CRYPT) || defined(NO_WOLFSSL_ESP32_CRYPT_AES) || \
 1287     defined(NEED_AES_HW_FALLBACK))
 1288static const FLASH_QUALIFIER word32 rcon[] = {
 1289    0x01000000, 0x02000000, 0x04000000, 0x08000000,
 1290    0x10000000, 0x20000000, 0x40000000, 0x80000000,
 1291    0x1B000000, 0x36000000,
 1292    /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
 1293};
 1294#endif /* ESP32 */
 1295#endif /* __aarch64__ || !WOLFSSL_ARMASM */
 1296
 1297#if !defined(WOLFSSL_ARMASM) || defined(WOLFSSL_AES_DIRECT) || \
 1298      defined(HAVE_AESCCM)
 1299#ifndef WOLFSSL_AES_SMALL_TABLES
 1300static const FLASH_QUALIFIER word32 Te[4][256] = {
 1301{
 1302    0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU,
 1303    0xfff2f20dU, 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U,
 1304    0x60303050U, 0x02010103U, 0xce6767a9U, 0x562b2b7dU,
 1305    0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, 0xec76769aU,
 1306    0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U,
 1307    0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU,
 1308    0x41adadecU, 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU,
 1309    0x239c9cbfU, 0x53a4a4f7U, 0xe4727296U, 0x9bc0c05bU,
 1310    0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, 0x4c26266aU,
 1311    0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU,
 1312    0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U,
 1313    0xe2717193U, 0xabd8d873U, 0x62313153U, 0x2a15153fU,
 1314    0x0804040cU, 0x95c7c752U, 0x46232365U, 0x9dc3c35eU,
 1315    0x30181828U, 0x379696a1U, 0x0a05050fU, 0x2f9a9ab5U,
 1316    0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU,
 1317    0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU,
 1318    0x1209091bU, 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU,
 1319    0x361b1b2dU, 0xdc6e6eb2U, 0xb45a5aeeU, 0x5ba0a0fbU,
 1320    0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, 0x7db3b3ceU,
 1321    0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U,
 1322    0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU,
 1323    0x40202060U, 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU,
 1324    0xd46a6abeU, 0x8dcbcb46U, 0x67bebed9U, 0x7239394bU,
 1325    0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, 0x85cfcf4aU,
 1326    0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U,
 1327    0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U,
 1328    0x8a4545cfU, 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U,
 1329    0xa05050f0U, 0x783c3c44U, 0x259f9fbaU, 0x4ba8a8e3U,
 1330    0xa25151f3U, 0x5da3a3feU, 0x804040c0U, 0x058f8f8aU,
 1331    0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U,
 1332    0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U,
 1333    0x20101030U, 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU,
 1334    0x81cdcd4cU, 0x180c0c14U, 0x26131335U, 0xc3ecec2fU,
 1335    0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, 0x2e171739U,
 1336    0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U,
 1337    0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U,
 1338    0xc06060a0U, 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU,
 1339    0x44222266U, 0x542a2a7eU, 0x3b9090abU, 0x0b888883U,
 1340    0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, 0x2814143cU,
 1341    0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U,
 1342    0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU,
 1343    0x924949dbU, 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U,
 1344    0x9fc2c25dU, 0xbdd3d36eU, 0x43acacefU, 0xc46262a6U,
 1345    0x399191a8U, 0x319595a4U, 0xd3e4e437U, 0xf279798bU,
 1346    0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U,
 1347    0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U,
 1348    0xd86c6cb4U, 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U,
 1349    0xca6565afU, 0xf47a7a8eU, 0x47aeaee9U, 0x10080818U,
 1350    0x6fbabad5U, 0xf0787888U, 0x4a25256fU, 0x5c2e2e72U,
 1351    0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U,
 1352    0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U,
 1353    0x964b4bddU, 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U,
 1354    0xe0707090U, 0x7c3e3e42U, 0x71b5b5c4U, 0xcc6666aaU,
 1355    0x904848d8U, 0x06030305U, 0xf7f6f601U, 0x1c0e0e12U,
 1356    0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U,
 1357    0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U,
 1358    0xd9e1e138U, 0xebf8f813U, 0x2b9898b3U, 0x22111133U,
 1359    0xd26969bbU, 0xa9d9d970U, 0x078e8e89U, 0x339494a7U,
 1360    0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, 0xc9e9e920U,
 1361    0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU,
 1362    0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U,
 1363    0x65bfbfdaU, 0xd7e6e631U, 0x844242c6U, 0xd06868b8U,
 1364    0x824141c3U, 0x299999b0U, 0x5a2d2d77U, 0x1e0f0f11U,
 1365    0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, 0x2c16163aU,
 1366},
 1367{
 1368    0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU,
 1369    0x0dfff2f2U, 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U,
 1370    0x50603030U, 0x03020101U, 0xa9ce6767U, 0x7d562b2bU,
 1371    0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, 0x9aec7676U,
 1372    0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU,
 1373    0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U,
 1374    0xec41adadU, 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU,
 1375    0xbf239c9cU, 0xf753a4a4U, 0x96e47272U, 0x5b9bc0c0U,
 1376    0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, 0x6a4c2626U,
 1377    0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU,
 1378    0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U,
 1379    0x93e27171U, 0x73abd8d8U, 0x53623131U, 0x3f2a1515U,
 1380    0x0c080404U, 0x5295c7c7U, 0x65462323U, 0x5e9dc3c3U,
 1381    0x28301818U, 0xa1379696U, 0x0f0a0505U, 0xb52f9a9aU,
 1382    0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U,
 1383    0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U,
 1384    0x1b120909U, 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU,
 1385    0x2d361b1bU, 0xb2dc6e6eU, 0xeeb45a5aU, 0xfb5ba0a0U,
 1386    0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, 0xce7db3b3U,
 1387    0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U,
 1388    0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU,
 1389    0x60402020U, 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU,
 1390    0xbed46a6aU, 0x468dcbcbU, 0xd967bebeU, 0x4b723939U,
 1391    0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, 0x4a85cfcfU,
 1392    0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU,
 1393    0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U,
 1394    0xcf8a4545U, 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU,
 1395    0xf0a05050U, 0x44783c3cU, 0xba259f9fU, 0xe34ba8a8U,
 1396    0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, 0x8a058f8fU,
 1397    0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U,
 1398    0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U,
 1399    0x30201010U, 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U,
 1400    0x4c81cdcdU, 0x14180c0cU, 0x35261313U, 0x2fc3ececU,
 1401    0xe1be5f5fU, 0xa2359797U, 0xcc884444U, 0x392e1717U,
 1402    0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU,
 1403    0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U,
 1404    0xa0c06060U, 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU,
 1405    0x66442222U, 0x7e542a2aU, 0xab3b9090U, 0x830b8888U,
 1406    0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, 0x3c281414U,
 1407    0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU,
 1408    0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU,
 1409    0xdb924949U, 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU,
 1410    0x5d9fc2c2U, 0x6ebdd3d3U, 0xef43acacU, 0xa6c46262U,
 1411    0xa8399191U, 0xa4319595U, 0x37d3e4e4U, 0x8bf27979U,
 1412    0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU,
 1413    0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U,
 1414    0xb4d86c6cU, 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU,
 1415    0xafca6565U, 0x8ef47a7aU, 0xe947aeaeU, 0x18100808U,
 1416    0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, 0x725c2e2eU,
 1417    0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U,
 1418    0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU,
 1419    0xdd964b4bU, 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU,
 1420    0x90e07070U, 0x427c3e3eU, 0xc471b5b5U, 0xaacc6666U,
 1421    0xd8904848U, 0x05060303U, 0x01f7f6f6U, 0x121c0e0eU,
 1422    0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U,
 1423    0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU,
 1424    0x38d9e1e1U, 0x13ebf8f8U, 0xb32b9898U, 0x33221111U,
 1425    0xbbd26969U, 0x70a9d9d9U, 0x89078e8eU, 0xa7339494U,
 1426    0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, 0x20c9e9e9U,
 1427    0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU,
 1428    0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU,
 1429    0xda65bfbfU, 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U,
 1430    0xc3824141U, 0xb0299999U, 0x775a2d2dU, 0x111e0f0fU,
 1431    0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, 0x3a2c1616U,
 1432},
 1433{
 1434    0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU,
 1435    0xf20dfff2U, 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U,
 1436    0x30506030U, 0x01030201U, 0x67a9ce67U, 0x2b7d562bU,
 1437    0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, 0x769aec76U,
 1438    0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU,
 1439    0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U,
 1440    0xadec41adU, 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU,
 1441    0x9cbf239cU, 0xa4f753a4U, 0x7296e472U, 0xc05b9bc0U,
 1442    0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, 0x266a4c26U,
 1443    0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU,
 1444    0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U,
 1445    0x7193e271U, 0xd873abd8U, 0x31536231U, 0x153f2a15U,
 1446    0x040c0804U, 0xc75295c7U, 0x23654623U, 0xc35e9dc3U,
 1447    0x18283018U, 0x96a13796U, 0x050f0a05U, 0x9ab52f9aU,
 1448    0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U,
 1449    0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U,
 1450    0x091b1209U, 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU,
 1451    0x1b2d361bU, 0x6eb2dc6eU, 0x5aeeb45aU, 0xa0fb5ba0U,
 1452    0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, 0xb3ce7db3U,
 1453    0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U,
 1454    0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU,
 1455    0x20604020U, 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU,
 1456    0x6abed46aU, 0xcb468dcbU, 0xbed967beU, 0x394b7239U,
 1457    0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, 0xcf4a85cfU,
 1458    0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU,
 1459    0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U,
 1460    0x45cf8a45U, 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU,
 1461    0x50f0a050U, 0x3c44783cU, 0x9fba259fU, 0xa8e34ba8U,
 1462    0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, 0x8f8a058fU,
 1463    0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U,
 1464    0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U,
 1465    0x10302010U, 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U,
 1466    0xcd4c81cdU, 0x0c14180cU, 0x13352613U, 0xec2fc3ecU,
 1467    0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, 0x17392e17U,
 1468    0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU,
 1469    0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U,
 1470    0x60a0c060U, 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU,
 1471    0x22664422U, 0x2a7e542aU, 0x90ab3b90U, 0x88830b88U,
 1472    0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, 0x143c2814U,
 1473    0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU,
 1474    0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU,
 1475    0x49db9249U, 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU,
 1476    0xc25d9fc2U, 0xd36ebdd3U, 0xacef43acU, 0x62a6c462U,
 1477    0x91a83991U, 0x95a43195U, 0xe437d3e4U, 0x798bf279U,
 1478    0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU,
 1479    0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U,
 1480    0x6cb4d86cU, 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU,
 1481    0x65afca65U, 0x7a8ef47aU, 0xaee947aeU, 0x08181008U,
 1482    0xbad56fbaU, 0x7888f078U, 0x256f4a25U, 0x2e725c2eU,
 1483    0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U,
 1484    0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU,
 1485    0x4bdd964bU, 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU,
 1486    0x7090e070U, 0x3e427c3eU, 0xb5c471b5U, 0x66aacc66U,
 1487    0x48d89048U, 0x03050603U, 0xf601f7f6U, 0x0e121c0eU,
 1488    0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U,
 1489    0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU,
 1490    0xe138d9e1U, 0xf813ebf8U, 0x98b32b98U, 0x11332211U,
 1491    0x69bbd269U, 0xd970a9d9U, 0x8e89078eU, 0x94a73394U,
 1492    0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, 0xe920c9e9U,
 1493    0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU,
 1494    0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU,
 1495    0xbfda65bfU, 0xe631d7e6U, 0x42c68442U, 0x68b8d068U,
 1496    0x41c38241U, 0x99b02999U, 0x2d775a2dU, 0x0f111e0fU,
 1497    0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, 0x163a2c16U,
 1498},
 1499{
 1500    0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U,
 1501    0xf2f20dffU, 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U,
 1502    0x30305060U, 0x01010302U, 0x6767a9ceU, 0x2b2b7d56U,
 1503    0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, 0x76769aecU,
 1504    0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU,
 1505    0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU,
 1506    0xadadec41U, 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U,
 1507    0x9c9cbf23U, 0xa4a4f753U, 0x727296e4U, 0xc0c05b9bU,
 1508    0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, 0x26266a4cU,
 1509    0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U,
 1510    0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U,
 1511    0x717193e2U, 0xd8d873abU, 0x31315362U, 0x15153f2aU,
 1512    0x04040c08U, 0xc7c75295U, 0x23236546U, 0xc3c35e9dU,
 1513    0x18182830U, 0x9696a137U, 0x05050f0aU, 0x9a9ab52fU,
 1514    0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU,
 1515    0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU,
 1516    0x09091b12U, 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U,
 1517    0x1b1b2d36U, 0x6e6eb2dcU, 0x5a5aeeb4U, 0xa0a0fb5bU,
 1518    0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, 0xb3b3ce7dU,
 1519    0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U,
 1520    0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U,
 1521    0x20206040U, 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U,
 1522    0x6a6abed4U, 0xcbcb468dU, 0xbebed967U, 0x39394b72U,
 1523    0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, 0xcfcf4a85U,
 1524    0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU,
 1525    0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U,
 1526    0x4545cf8aU, 0xf9f910e9U, 0x02020604U, 0x7f7f81feU,
 1527    0x5050f0a0U, 0x3c3c4478U, 0x9f9fba25U, 0xa8a8e34bU,
 1528    0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, 0x8f8f8a05U,
 1529    0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U,
 1530    0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U,
 1531    0x10103020U, 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU,
 1532    0xcdcd4c81U, 0x0c0c1418U, 0x13133526U, 0xecec2fc3U,
 1533    0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, 0x1717392eU,
 1534    0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU,
 1535    0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U,
 1536    0x6060a0c0U, 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U,
 1537    0x22226644U, 0x2a2a7e54U, 0x9090ab3bU, 0x8888830bU,
 1538    0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, 0x14143c28U,
 1539    0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU,
 1540    0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U,
 1541    0x4949db92U, 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U,
 1542    0xc2c25d9fU, 0xd3d36ebdU, 0xacacef43U, 0x6262a6c4U,
 1543    0x9191a839U, 0x9595a431U, 0xe4e437d3U, 0x79798bf2U,
 1544    0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU,
 1545    0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U,
 1546    0x6c6cb4d8U, 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU,
 1547    0x6565afcaU, 0x7a7a8ef4U, 0xaeaee947U, 0x08081810U,
 1548    0xbabad56fU, 0x787888f0U, 0x25256f4aU, 0x2e2e725cU,
 1549    0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U,
 1550    0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU,
 1551    0x4b4bdd96U, 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU,
 1552    0x707090e0U, 0x3e3e427cU, 0xb5b5c471U, 0x6666aaccU,
 1553    0x4848d890U, 0x03030506U, 0xf6f601f7U, 0x0e0e121cU,
 1554    0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U,
 1555    0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U,
 1556    0xe1e138d9U, 0xf8f813ebU, 0x9898b32bU, 0x11113322U,
 1557    0x6969bbd2U, 0xd9d970a9U, 0x8e8e8907U, 0x9494a733U,
 1558    0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, 0xe9e920c9U,
 1559    0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U,
 1560    0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU,
 1561    0xbfbfda65U, 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U,
 1562    0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU,
 1563    0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU,
 1564}
 1565};
 1566
 1567#ifdef HAVE_AES_DECRYPT
 1568#if defined(__aarch64__) || !defined(WOLFSSL_ARMASM)
 1569static const FLASH_QUALIFIER word32 Td[4][256] = {
 1570{
 1571    0x51f4a750U, 0x7e416553U, 0x1a17a4c3U, 0x3a275e96U,
 1572    0x3bab6bcbU, 0x1f9d45f1U, 0xacfa58abU, 0x4be30393U,
 1573    0x2030fa55U, 0xad766df6U, 0x88cc7691U, 0xf5024c25U,
 1574    0x4fe5d7fcU, 0xc52acbd7U, 0x26354480U, 0xb562a38fU,
 1575    0xdeb15a49U, 0x25ba1b67U, 0x45ea0e98U, 0x5dfec0e1U,
 1576    0xc32f7502U, 0x814cf012U, 0x8d4697a3U, 0x6bd3f9c6U,
 1577    0x038f5fe7U, 0x15929c95U, 0xbf6d7aebU, 0x955259daU,
 1578    0xd4be832dU, 0x587421d3U, 0x49e06929U, 0x8ec9c844U,
 1579    0x75c2896aU, 0xf48e7978U, 0x99583e6bU, 0x27b971ddU,
 1580    0xbee14fb6U, 0xf088ad17U, 0xc920ac66U, 0x7dce3ab4U,
 1581    0x63df4a18U, 0xe51a3182U, 0x97513360U, 0x62537f45U,
 1582    0xb16477e0U, 0xbb6bae84U, 0xfe81a01cU, 0xf9082b94U,
 1583    0x70486858U, 0x8f45fd19U, 0x94de6c87U, 0x527bf8b7U,
 1584    0xab73d323U, 0x724b02e2U, 0xe31f8f57U, 0x6655ab2aU,
 1585    0xb2eb2807U, 0x2fb5c203U, 0x86c57b9aU, 0xd33708a5U,
 1586    0x302887f2U, 0x23bfa5b2U, 0x02036abaU, 0xed16825cU,
 1587    0x8acf1c2bU, 0xa779b492U, 0xf307f2f0U, 0x4e69e2a1U,
 1588    0x65daf4cdU, 0x0605bed5U, 0xd134621fU, 0xc4a6fe8aU,
 1589    0x342e539dU, 0xa2f355a0U, 0x058ae132U, 0xa4f6eb75U,
 1590    0x0b83ec39U, 0x4060efaaU, 0x5e719f06U, 0xbd6e1051U,
 1591    0x3e218af9U, 0x96dd063dU, 0xdd3e05aeU, 0x4de6bd46U,
 1592    0x91548db5U, 0x71c45d05U, 0x0406d46fU, 0x605015ffU,
 1593    0x1998fb24U, 0xd6bde997U, 0x894043ccU, 0x67d99e77U,
 1594    0xb0e842bdU, 0x07898b88U, 0xe7195b38U, 0x79c8eedbU,
 1595    0xa17c0a47U, 0x7c420fe9U, 0xf8841ec9U, 0x00000000U,
 1596    0x09808683U, 0x322bed48U, 0x1e1170acU, 0x6c5a724eU,
 1597    0xfd0efffbU, 0x0f853856U, 0x3daed51eU, 0x362d3927U,
 1598    0x0a0fd964U, 0x685ca621U, 0x9b5b54d1U, 0x24362e3aU,
 1599    0x0c0a67b1U, 0x9357e70fU, 0xb4ee96d2U, 0x1b9b919eU,
 1600    0x80c0c54fU, 0x61dc20a2U, 0x5a774b69U, 0x1c121a16U,
 1601    0xe293ba0aU, 0xc0a02ae5U, 0x3c22e043U, 0x121b171dU,
 1602    0x0e090d0bU, 0xf28bc7adU, 0x2db6a8b9U, 0x141ea9c8U,
 1603    0x57f11985U, 0xaf75074cU, 0xee99ddbbU, 0xa37f60fdU,
 1604    0xf701269fU, 0x5c72f5bcU, 0x44663bc5U, 0x5bfb7e34U,
 1605    0x8b432976U, 0xcb23c6dcU, 0xb6edfc68U, 0xb8e4f163U,
 1606    0xd731dccaU, 0x42638510U, 0x13972240U, 0x84c61120U,
 1607    0x854a247dU, 0xd2bb3df8U, 0xaef93211U, 0xc729a16dU,
 1608    0x1d9e2f4bU, 0xdcb230f3U, 0x0d8652ecU, 0x77c1e3d0U,
 1609    0x2bb3166cU, 0xa970b999U, 0x119448faU, 0x47e96422U,
 1610    0xa8fc8cc4U, 0xa0f03f1aU, 0x567d2cd8U, 0x223390efU,
 1611    0x87494ec7U, 0xd938d1c1U, 0x8ccaa2feU, 0x98d40b36U,
 1612    0xa6f581cfU, 0xa57ade28U, 0xdab78e26U, 0x3fadbfa4U,
 1613    0x2c3a9de4U, 0x5078920dU, 0x6a5fcc9bU, 0x547e4662U,
 1614    0xf68d13c2U, 0x90d8b8e8U, 0x2e39f75eU, 0x82c3aff5U,
 1615    0x9f5d80beU, 0x69d0937cU, 0x6fd52da9U, 0xcf2512b3U,
 1616    0xc8ac993bU, 0x10187da7U, 0xe89c636eU, 0xdb3bbb7bU,
 1617    0xcd267809U, 0x6e5918f4U, 0xec9ab701U, 0x834f9aa8U,
 1618    0xe6956e65U, 0xaaffe67eU, 0x21bccf08U, 0xef15e8e6U,
 1619    0xbae79bd9U, 0x4a6f36ceU, 0xea9f09d4U, 0x29b07cd6U,
 1620    0x31a4b2afU, 0x2a3f2331U, 0xc6a59430U, 0x35a266c0U,
 1621    0x744ebc37U, 0xfc82caa6U, 0xe090d0b0U, 0x33a7d815U,
 1622    0xf104984aU, 0x41ecdaf7U, 0x7fcd500eU, 0x1791f62fU,
 1623    0x764dd68dU, 0x43efb04dU, 0xccaa4d54U, 0xe49604dfU,
 1624    0x9ed1b5e3U, 0x4c6a881bU, 0xc12c1fb8U, 0x4665517fU,
 1625    0x9d5eea04U, 0x018c355dU, 0xfa877473U, 0xfb0b412eU,
 1626    0xb3671d5aU, 0x92dbd252U, 0xe9105633U, 0x6dd64713U,
 1627    0x9ad7618cU, 0x37a10c7aU, 0x59f8148eU, 0xeb133c89U,
 1628    0xcea927eeU, 0xb761c935U, 0xe11ce5edU, 0x7a47b13cU,
 1629    0x9cd2df59U, 0x55f2733fU, 0x1814ce79U, 0x73c737bfU,
 1630    0x53f7cdeaU, 0x5ffdaa5bU, 0xdf3d6f14U, 0x7844db86U,
 1631    0xcaaff381U, 0xb968c43eU, 0x3824342cU, 0xc2a3405fU,
 1632    0x161dc372U, 0xbce2250cU, 0x283c498bU, 0xff0d9541U,
 1633    0x39a80171U, 0x080cb3deU, 0xd8b4e49cU, 0x6456c190U,
 1634    0x7bcb8461U, 0xd532b670U, 0x486c5c74U, 0xd0b85742U,
 1635},
 1636{
 1637    0x5051f4a7U, 0x537e4165U, 0xc31a17a4U, 0x963a275eU,
 1638    0xcb3bab6bU, 0xf11f9d45U, 0xabacfa58U, 0x934be303U,
 1639    0x552030faU, 0xf6ad766dU, 0x9188cc76U, 0x25f5024cU,
 1640    0xfc4fe5d7U, 0xd7c52acbU, 0x80263544U, 0x8fb562a3U,
 1641    0x49deb15aU, 0x6725ba1bU, 0x9845ea0eU, 0xe15dfec0U,
 1642    0x02c32f75U, 0x12814cf0U, 0xa38d4697U, 0xc66bd3f9U,
 1643    0xe7038f5fU, 0x9515929cU, 0xebbf6d7aU, 0xda955259U,
 1644    0x2dd4be83U, 0xd3587421U, 0x2949e069U, 0x448ec9c8U,
 1645    0x6a75c289U, 0x78f48e79U, 0x6b99583eU, 0xdd27b971U,
 1646    0xb6bee14fU, 0x17f088adU, 0x66c920acU, 0xb47dce3aU,
 1647    0x1863df4aU, 0x82e51a31U, 0x60975133U, 0x4562537fU,
 1648    0xe0b16477U, 0x84bb6baeU, 0x1cfe81a0U, 0x94f9082bU,
 1649    0x58704868U, 0x198f45fdU, 0x8794de6cU, 0xb7527bf8U,
 1650    0x23ab73d3U, 0xe2724b02U, 0x57e31f8fU, 0x2a6655abU,
 1651    0x07b2eb28U, 0x032fb5c2U, 0x9a86c57bU, 0xa5d33708U,
 1652    0xf2302887U, 0xb223bfa5U, 0xba02036aU, 0x5ced1682U,
 1653    0x2b8acf1cU, 0x92a779b4U, 0xf0f307f2U, 0xa14e69e2U,
 1654    0xcd65daf4U, 0xd50605beU, 0x1fd13462U, 0x8ac4a6feU,
 1655    0x9d342e53U, 0xa0a2f355U, 0x32058ae1U, 0x75a4f6ebU,
 1656    0x390b83ecU, 0xaa4060efU, 0x065e719fU, 0x51bd6e10U,
 1657    0xf93e218aU, 0x3d96dd06U, 0xaedd3e05U, 0x464de6bdU,
 1658    0xb591548dU, 0x0571c45dU, 0x6f0406d4U, 0xff605015U,
 1659    0x241998fbU, 0x97d6bde9U, 0xcc894043U, 0x7767d99eU,
 1660    0xbdb0e842U, 0x8807898bU, 0x38e7195bU, 0xdb79c8eeU,
 1661    0x47a17c0aU, 0xe97c420fU, 0xc9f8841eU, 0x00000000U,
 1662    0x83098086U, 0x48322bedU, 0xac1e1170U, 0x4e6c5a72U,
 1663    0xfbfd0effU, 0x560f8538U, 0x1e3daed5U, 0x27362d39U,
 1664    0x640a0fd9U, 0x21685ca6U, 0xd19b5b54U, 0x3a24362eU,
 1665    0xb10c0a67U, 0x0f9357e7U, 0xd2b4ee96U, 0x9e1b9b91U,
 1666    0x4f80c0c5U, 0xa261dc20U, 0x695a774bU, 0x161c121aU,
 1667    0x0ae293baU, 0xe5c0a02aU, 0x433c22e0U, 0x1d121b17U,
 1668    0x0b0e090dU, 0xadf28bc7U, 0xb92db6a8U, 0xc8141ea9U,
 1669    0x8557f119U, 0x4caf7507U, 0xbbee99ddU, 0xfda37f60U,
 1670    0x9ff70126U, 0xbc5c72f5U, 0xc544663bU, 0x345bfb7eU,
 1671    0x768b4329U, 0xdccb23c6U, 0x68b6edfcU, 0x63b8e4f1U,
 1672    0xcad731dcU, 0x10426385U, 0x40139722U, 0x2084c611U,
 1673    0x7d854a24U, 0xf8d2bb3dU, 0x11aef932U, 0x6dc729a1U,
 1674    0x4b1d9e2fU, 0xf3dcb230U, 0xec0d8652U, 0xd077c1e3U,
 1675    0x6c2bb316U, 0x99a970b9U, 0xfa119448U, 0x2247e964U,
 1676    0xc4a8fc8cU, 0x1aa0f03fU, 0xd8567d2cU, 0xef223390U,
 1677    0xc787494eU, 0xc1d938d1U, 0xfe8ccaa2U, 0x3698d40bU,
 1678    0xcfa6f581U, 0x28a57adeU, 0x26dab78eU, 0xa43fadbfU,
 1679    0xe42c3a9dU, 0x0d507892U, 0x9b6a5fccU, 0x62547e46U,
 1680    0xc2f68d13U, 0xe890d8b8U, 0x5e2e39f7U, 0xf582c3afU,
 1681    0xbe9f5d80U, 0x7c69d093U, 0xa96fd52dU, 0xb3cf2512U,
 1682    0x3bc8ac99U, 0xa710187dU, 0x6ee89c63U, 0x7bdb3bbbU,
 1683    0x09cd2678U, 0xf46e5918U, 0x01ec9ab7U, 0xa8834f9aU,
 1684    0x65e6956eU, 0x7eaaffe6U, 0x0821bccfU, 0xe6ef15e8U,
 1685    0xd9bae79bU, 0xce4a6f36U, 0xd4ea9f09U, 0xd629b07cU,
 1686    0xaf31a4b2U, 0x312a3f23U, 0x30c6a594U, 0xc035a266U,
 1687    0x37744ebcU, 0xa6fc82caU, 0xb0e090d0U, 0x1533a7d8U,
 1688    0x4af10498U, 0xf741ecdaU, 0x0e7fcd50U, 0x2f1791f6U,
 1689    0x8d764dd6U, 0x4d43efb0U, 0x54ccaa4dU, 0xdfe49604U,
 1690    0xe39ed1b5U, 0x1b4c6a88U, 0xb8c12c1fU, 0x7f466551U,
 1691    0x049d5eeaU, 0x5d018c35U, 0x73fa8774U, 0x2efb0b41U,
 1692    0x5ab3671dU, 0x5292dbd2U, 0x33e91056U, 0x136dd647U,
 1693    0x8c9ad761U, 0x7a37a10cU, 0x8e59f814U, 0x89eb133cU,
 1694    0xeecea927U, 0x35b761c9U, 0xede11ce5U, 0x3c7a47b1U,
 1695    0x599cd2dfU, 0x3f55f273U, 0x791814ceU, 0xbf73c737U,
 1696    0xea53f7cdU, 0x5b5ffdaaU, 0x14df3d6fU, 0x867844dbU,
 1697    0x81caaff3U, 0x3eb968c4U, 0x2c382434U, 0x5fc2a340U,
 1698    0x72161dc3U, 0x0cbce225U, 0x8b283c49U, 0x41ff0d95U,
 1699    0x7139a801U, 0xde080cb3U, 0x9cd8b4e4U, 0x906456c1U,
 1700    0x617bcb84U, 0x70d532b6U, 0x74486c5cU, 0x42d0b857U,
 1701},
 1702{
 1703    0xa75051f4U, 0x65537e41U, 0xa4c31a17U, 0x5e963a27U,
 1704    0x6bcb3babU, 0x45f11f9dU, 0x58abacfaU, 0x03934be3U,
 1705    0xfa552030U, 0x6df6ad76U, 0x769188ccU, 0x4c25f502U,
 1706    0xd7fc4fe5U, 0xcbd7c52aU, 0x44802635U, 0xa38fb562U,
 1707    0x5a49deb1U, 0x1b6725baU, 0x0e9845eaU, 0xc0e15dfeU,
 1708    0x7502c32fU, 0xf012814cU, 0x97a38d46U, 0xf9c66bd3U,
 1709    0x5fe7038fU, 0x9c951592U, 0x7aebbf6dU, 0x59da9552U,
 1710    0x832dd4beU, 0x21d35874U, 0x692949e0U, 0xc8448ec9U,
 1711    0x896a75c2U, 0x7978f48eU, 0x3e6b9958U, 0x71dd27b9U,
 1712    0x4fb6bee1U, 0xad17f088U, 0xac66c920U, 0x3ab47dceU,
 1713    0x4a1863dfU, 0x3182e51aU, 0x33609751U, 0x7f456253U,
 1714    0x77e0b164U, 0xae84bb6bU, 0xa01cfe81U, 0x2b94f908U,
 1715    0x68587048U, 0xfd198f45U, 0x6c8794deU, 0xf8b7527bU,
 1716    0xd323ab73U, 0x02e2724bU, 0x8f57e31fU, 0xab2a6655U,
 1717    0x2807b2ebU, 0xc2032fb5U, 0x7b9a86c5U, 0x08a5d337U,
 1718    0x87f23028U, 0xa5b223bfU, 0x6aba0203U, 0x825ced16U,
 1719    0x1c2b8acfU, 0xb492a779U, 0xf2f0f307U, 0xe2a14e69U,
 1720    0xf4cd65daU, 0xbed50605U, 0x621fd134U, 0xfe8ac4a6U,
 1721    0x539d342eU, 0x55a0a2f3U, 0xe132058aU, 0xeb75a4f6U,
 1722    0xec390b83U, 0xefaa4060U, 0x9f065e71U, 0x1051bd6eU,
 1723
 1724    0x8af93e21U, 0x063d96ddU, 0x05aedd3eU, 0xbd464de6U,
 1725    0x8db59154U, 0x5d0571c4U, 0xd46f0406U, 0x15ff6050U,
 1726    0xfb241998U, 0xe997d6bdU, 0x43cc8940U, 0x9e7767d9U,
 1727    0x42bdb0e8U, 0x8b880789U, 0x5b38e719U, 0xeedb79c8U,
 1728    0x0a47a17cU, 0x0fe97c42U, 0x1ec9f884U, 0x00000000U,
 1729    0x86830980U, 0xed48322bU, 0x70ac1e11U, 0x724e6c5aU,
 1730    0xfffbfd0eU, 0x38560f85U, 0xd51e3daeU, 0x3927362dU,
 1731    0xd9640a0fU, 0xa621685cU, 0x54d19b5bU, 0x2e3a2436U,
 1732    0x67b10c0aU, 0xe70f9357U, 0x96d2b4eeU, 0x919e1b9bU,
 1733    0xc54f80c0U, 0x20a261dcU, 0x4b695a77U, 0x1a161c12U,
 1734    0xba0ae293U, 0x2ae5c0a0U, 0xe0433c22U, 0x171d121bU,
 1735    0x0d0b0e09U, 0xc7adf28bU, 0xa8b92db6U, 0xa9c8141eU,
 1736    0x198557f1U, 0x074caf75U, 0xddbbee99U, 0x60fda37fU,
 1737    0x269ff701U, 0xf5bc5c72U, 0x3bc54466U, 0x7e345bfbU,
 1738    0x29768b43U, 0xc6dccb23U, 0xfc68b6edU, 0xf163b8e4U,
 1739    0xdccad731U, 0x85104263U, 0x22401397U, 0x112084c6U,
 1740    0x247d854aU, 0x3df8d2bbU, 0x3211aef9U, 0xa16dc729U,
 1741    0x2f4b1d9eU, 0x30f3dcb2U, 0x52ec0d86U, 0xe3d077c1U,
 1742    0x166c2bb3U, 0xb999a970U, 0x48fa1194U, 0x642247e9U,
 1743    0x8cc4a8fcU, 0x3f1aa0f0U, 0x2cd8567dU, 0x90ef2233U,
 1744    0x4ec78749U, 0xd1c1d938U, 0xa2fe8ccaU, 0x0b3698d4U,
 1745    0x81cfa6f5U, 0xde28a57aU, 0x8e26dab7U, 0xbfa43fadU,
 1746    0x9de42c3aU, 0x920d5078U, 0xcc9b6a5fU, 0x4662547eU,
 1747    0x13c2f68dU, 0xb8e890d8U, 0xf75e2e39U, 0xaff582c3U,
 1748    0x80be9f5dU, 0x937c69d0U, 0x2da96fd5U, 0x12b3cf25U,
 1749    0x993bc8acU, 0x7da71018U, 0x636ee89cU, 0xbb7bdb3bU,
 1750    0x7809cd26U, 0x18f46e59U, 0xb701ec9aU, 0x9aa8834fU,
 1751    0x6e65e695U, 0xe67eaaffU, 0xcf0821bcU, 0xe8e6ef15U,
 1752    0x9bd9bae7U, 0x36ce4a6fU, 0x09d4ea9fU, 0x7cd629b0U,
 1753    0xb2af31a4U, 0x23312a3fU, 0x9430c6a5U, 0x66c035a2U,
 1754    0xbc37744eU, 0xcaa6fc82U, 0xd0b0e090U, 0xd81533a7U,
 1755    0x984af104U, 0xdaf741ecU, 0x500e7fcdU, 0xf62f1791U,
 1756    0xd68d764dU, 0xb04d43efU, 0x4d54ccaaU, 0x04dfe496U,
 1757    0xb5e39ed1U, 0x881b4c6aU, 0x1fb8c12cU, 0x517f4665U,
 1758    0xea049d5eU, 0x355d018cU, 0x7473fa87U, 0x412efb0bU,
 1759    0x1d5ab367U, 0xd25292dbU, 0x5633e910U, 0x47136dd6U,
 1760    0x618c9ad7U, 0x0c7a37a1U, 0x148e59f8U, 0x3c89eb13U,
 1761    0x27eecea9U, 0xc935b761U, 0xe5ede11cU, 0xb13c7a47U,
 1762    0xdf599cd2U, 0x733f55f2U, 0xce791814U, 0x37bf73c7U,
 1763    0xcdea53f7U, 0xaa5b5ffdU, 0x6f14df3dU, 0xdb867844U,
 1764    0xf381caafU, 0xc43eb968U, 0x342c3824U, 0x405fc2a3U,
 1765    0xc372161dU, 0x250cbce2U, 0x498b283cU, 0x9541ff0dU,
 1766    0x017139a8U, 0xb3de080cU, 0xe49cd8b4U, 0xc1906456U,
 1767    0x84617bcbU, 0xb670d532U, 0x5c74486cU, 0x5742d0b8U,
 1768},
 1769{
 1770    0xf4a75051U, 0x4165537eU, 0x17a4c31aU, 0x275e963aU,
 1771    0xab6bcb3bU, 0x9d45f11fU, 0xfa58abacU, 0xe303934bU,
 1772    0x30fa5520U, 0x766df6adU, 0xcc769188U, 0x024c25f5U,
 1773    0xe5d7fc4fU, 0x2acbd7c5U, 0x35448026U, 0x62a38fb5U,
 1774    0xb15a49deU, 0xba1b6725U, 0xea0e9845U, 0xfec0e15dU,
 1775    0x2f7502c3U, 0x4cf01281U, 0x4697a38dU, 0xd3f9c66bU,
 1776    0x8f5fe703U, 0x929c9515U, 0x6d7aebbfU, 0x5259da95U,
 1777    0xbe832dd4U, 0x7421d358U, 0xe0692949U, 0xc9c8448eU,
 1778    0xc2896a75U, 0x8e7978f4U, 0x583e6b99U, 0xb971dd27U,
 1779    0xe14fb6beU, 0x88ad17f0U, 0x20ac66c9U, 0xce3ab47dU,
 1780    0xdf4a1863U, 0x1a3182e5U, 0x51336097U, 0x537f4562U,
 1781    0x6477e0b1U, 0x6bae84bbU, 0x81a01cfeU, 0x082b94f9U,
 1782    0x48685870U, 0x45fd198fU, 0xde6c8794U, 0x7bf8b752U,
 1783    0x73d323abU, 0x4b02e272U, 0x1f8f57e3U, 0x55ab2a66U,
 1784    0xeb2807b2U, 0xb5c2032fU, 0xc57b9a86U, 0x3708a5d3U,
 1785    0x2887f230U, 0xbfa5b223U, 0x036aba02U, 0x16825cedU,
 1786    0xcf1c2b8aU, 0x79b492a7U, 0x07f2f0f3U, 0x69e2a14eU,
 1787    0xdaf4cd65U, 0x05bed506U, 0x34621fd1U, 0xa6fe8ac4U,
 1788    0x2e539d34U, 0xf355a0a2U, 0x8ae13205U, 0xf6eb75a4U,
 1789    0x83ec390bU, 0x60efaa40U, 0x719f065eU, 0x6e1051bdU,
 1790    0x218af93eU, 0xdd063d96U, 0x3e05aeddU, 0xe6bd464dU,
 1791    0x548db591U, 0xc45d0571U, 0x06d46f04U, 0x5015ff60U,
 1792    0x98fb2419U, 0xbde997d6U, 0x4043cc89U, 0xd99e7767U,
 1793    0xe842bdb0U, 0x898b8807U, 0x195b38e7U, 0xc8eedb79U,
 1794    0x7c0a47a1U, 0x420fe97cU, 0x841ec9f8U, 0x00000000U,
 1795    0x80868309U, 0x2bed4832U, 0x1170ac1eU, 0x5a724e6cU,
 1796    0x0efffbfdU, 0x8538560fU, 0xaed51e3dU, 0x2d392736U,
 1797    0x0fd9640aU, 0x5ca62168U, 0x5b54d19bU, 0x362e3a24U,
 1798    0x0a67b10cU, 0x57e70f93U, 0xee96d2b4U, 0x9b919e1bU,
 1799    0xc0c54f80U, 0xdc20a261U, 0x774b695aU, 0x121a161cU,
 1800    0x93ba0ae2U, 0xa02ae5c0U, 0x22e0433cU, 0x1b171d12U,
 1801    0x090d0b0eU, 0x8bc7adf2U, 0xb6a8b92dU, 0x1ea9c814U,
 1802    0xf1198557U, 0x75074cafU, 0x99ddbbeeU, 0x7f60fda3U,
 1803    0x01269ff7U, 0x72f5bc5cU, 0x663bc544U, 0xfb7e345bU,
 1804    0x4329768bU, 0x23c6dccbU, 0xedfc68b6U, 0xe4f163b8U,
 1805    0x31dccad7U, 0x63851042U, 0x97224013U, 0xc6112084U,
 1806    0x4a247d85U, 0xbb3df8d2U, 0xf93211aeU, 0x29a16dc7U,
 1807    0x9e2f4b1dU, 0xb230f3dcU, 0x8652ec0dU, 0xc1e3d077U,
 1808    0xb3166c2bU, 0x70b999a9U, 0x9448fa11U, 0xe9642247U,
 1809    0xfc8cc4a8U, 0xf03f1aa0U, 0x7d2cd856U, 0x3390ef22U,
 1810    0x494ec787U, 0x38d1c1d9U, 0xcaa2fe8cU, 0xd40b3698U,
 1811    0xf581cfa6U, 0x7ade28a5U, 0xb78e26daU, 0xadbfa43fU,
 1812    0x3a9de42cU, 0x78920d50U, 0x5fcc9b6aU, 0x7e466254U,
 1813    0x8d13c2f6U, 0xd8b8e890U, 0x39f75e2eU, 0xc3aff582U,
 1814    0x5d80be9fU, 0xd0937c69U, 0xd52da96fU, 0x2512b3cfU,
 1815    0xac993bc8U, 0x187da710U, 0x9c636ee8U, 0x3bbb7bdbU,
 1816    0x267809cdU, 0x5918f46eU, 0x9ab701ecU, 0x4f9aa883U,
 1817    0x956e65e6U, 0xffe67eaaU, 0xbccf0821U, 0x15e8e6efU,
 1818    0xe79bd9baU, 0x6f36ce4aU, 0x9f09d4eaU, 0xb07cd629U,
 1819    0xa4b2af31U, 0x3f23312aU, 0xa59430c6U, 0xa266c035U,
 1820    0x4ebc3774U, 0x82caa6fcU, 0x90d0b0e0U, 0xa7d81533U,
 1821    0x04984af1U, 0xecdaf741U, 0xcd500e7fU, 0x91f62f17U,
 1822    0x4dd68d76U, 0xefb04d43U, 0xaa4d54ccU, 0x9604dfe4U,
 1823    0xd1b5e39eU, 0x6a881b4cU, 0x2c1fb8c1U, 0x65517f46U,
 1824    0x5eea049dU, 0x8c355d01U, 0x877473faU, 0x0b412efbU,
 1825    0x671d5ab3U, 0xdbd25292U, 0x105633e9U, 0xd647136dU,
 1826    0xd7618c9aU, 0xa10c7a37U, 0xf8148e59U, 0x133c89ebU,
 1827    0xa927eeceU, 0x61c935b7U, 0x1ce5ede1U, 0x47b13c7aU,
 1828    0xd2df599cU, 0xf2733f55U, 0x14ce7918U, 0xc737bf73U,
 1829    0xf7cdea53U, 0xfdaa5b5fU, 0x3d6f14dfU, 0x44db8678U,
 1830    0xaff381caU, 0x68c43eb9U, 0x24342c38U, 0xa3405fc2U,
 1831    0x1dc37216U, 0xe2250cbcU, 0x3c498b28U, 0x0d9541ffU,
 1832    0xa8017139U, 0x0cb3de08U, 0xb4e49cd8U, 0x56c19064U,
 1833    0xcb84617bU, 0x32b670d5U, 0x6c5c7448U, 0xb85742d0U,
 1834}
 1835};
 1836#endif /* __aarch64__ || !WOLFSSL_ARMASM */
 1837#endif /* HAVE_AES_DECRYPT */
 1838#endif /* WOLFSSL_AES_SMALL_TABLES */
 1839
 1840#ifdef HAVE_AES_DECRYPT
 1841#if (defined(HAVE_AES_CBC) && !defined(WOLFSSL_DEVCRYPTO_CBC)) || \
 1842     defined(HAVE_AES_ECB) || defined(WOLFSSL_AES_DIRECT)
 1843#if defined(__aarch64__) || !defined(WOLFSSL_ARMASM)
 1844static const FLASH_QUALIFIER byte Td4[256] =
 1845{
 1846    0x52U, 0x09U, 0x6aU, 0xd5U, 0x30U, 0x36U, 0xa5U, 0x38U,
 1847    0xbfU, 0x40U, 0xa3U, 0x9eU, 0x81U, 0xf3U, 0xd7U, 0xfbU,
 1848    0x7cU, 0xe3U, 0x39U, 0x82U, 0x9bU, 0x2fU, 0xffU, 0x87U,
 1849    0x34U, 0x8eU, 0x43U, 0x44U, 0xc4U, 0xdeU, 0xe9U, 0xcbU,
 1850    0x54U, 0x7bU, 0x94U, 0x32U, 0xa6U, 0xc2U, 0x23U, 0x3dU,
 1851    0xeeU, 0x4cU, 0x95U, 0x0bU, 0x42U, 0xfaU, 0xc3U, 0x4eU,
 1852    0x08U, 0x2eU, 0xa1U, 0x66U, 0x28U, 0xd9U, 0x24U, 0xb2U,
 1853    0x76U, 0x5bU, 0xa2U, 0x49U, 0x6dU, 0x8bU, 0xd1U, 0x25U,
 1854    0x72U, 0xf8U, 0xf6U, 0x64U, 0x86U, 0x68U, 0x98U, 0x16U,
 1855    0xd4U, 0xa4U, 0x5cU, 0xccU, 0x5dU, 0x65U, 0xb6U, 0x92U,
 1856    0x6cU, 0x70U, 0x48U, 0x50U, 0xfdU, 0xedU, 0xb9U, 0xdaU,
 1857    0x5eU, 0x15U, 0x46U, 0x57U, 0xa7U, 0x8dU, 0x9dU, 0x84U,
 1858    0x90U, 0xd8U, 0xabU, 0x00U, 0x8cU, 0xbcU, 0xd3U, 0x0aU,
 1859    0xf7U, 0xe4U, 0x58U, 0x05U, 0xb8U, 0xb3U, 0x45U, 0x06U,
 1860    0xd0U, 0x2cU, 0x1eU, 0x8fU, 0xcaU, 0x3fU, 0x0fU, 0x02U,
 1861    0xc1U, 0xafU, 0xbdU, 0x03U, 0x01U, 0x13U, 0x8aU, 0x6bU,
 1862    0x3aU, 0x91U, 0x11U, 0x41U, 0x4fU, 0x67U, 0xdcU, 0xeaU,
 1863    0x97U, 0xf2U, 0xcfU, 0xceU, 0xf0U, 0xb4U, 0xe6U, 0x73U,
 1864    0x96U, 0xacU, 0x74U, 0x22U, 0xe7U, 0xadU, 0x35U, 0x85U,
 1865    0xe2U, 0xf9U, 0x37U, 0xe8U, 0x1cU, 0x75U, 0xdfU, 0x6eU,
 1866    0x47U, 0xf1U, 0x1aU, 0x71U, 0x1dU, 0x29U, 0xc5U, 0x89U,
 1867    0x6fU, 0xb7U, 0x62U, 0x0eU, 0xaaU, 0x18U, 0xbeU, 0x1bU,
 1868    0xfcU, 0x56U, 0x3eU, 0x4bU, 0xc6U, 0xd2U, 0x79U, 0x20U,
 1869    0x9aU, 0xdbU, 0xc0U, 0xfeU, 0x78U, 0xcdU, 0x5aU, 0xf4U,
 1870    0x1fU, 0xddU, 0xa8U, 0x33U, 0x88U, 0x07U, 0xc7U, 0x31U,
 1871    0xb1U, 0x12U, 0x10U, 0x59U, 0x27U, 0x80U, 0xecU, 0x5fU,
 1872    0x60U, 0x51U, 0x7fU, 0xa9U, 0x19U, 0xb5U, 0x4aU, 0x0dU,
 1873    0x2dU, 0xe5U, 0x7aU, 0x9fU, 0x93U, 0xc9U, 0x9cU, 0xefU,
 1874    0xa0U, 0xe0U, 0x3bU, 0x4dU, 0xaeU, 0x2aU, 0xf5U, 0xb0U,
 1875    0xc8U, 0xebU, 0xbbU, 0x3cU, 0x83U, 0x53U, 0x99U, 0x61U,
 1876    0x17U, 0x2bU, 0x04U, 0x7eU, 0xbaU, 0x77U, 0xd6U, 0x26U,
 1877    0xe1U, 0x69U, 0x14U, 0x63U, 0x55U, 0x21U, 0x0cU, 0x7dU,
 1878};
 1879#endif
 1880#endif /* HAVE_AES_CBC || WOLFSSL_AES_DIRECT */
 1881#endif /* HAVE_AES_DECRYPT */
 1882
 1883#define GETBYTE(x, y) (word32)((byte)((x) >> (8 * (y))))
 1884
 1885#ifdef WOLFSSL_AES_SMALL_TABLES
 1886static const byte Tsbox[256] = {
 1887    0x63U, 0x7cU, 0x77U, 0x7bU, 0xf2U, 0x6bU, 0x6fU, 0xc5U,
 1888    0x30U, 0x01U, 0x67U, 0x2bU, 0xfeU, 0xd7U, 0xabU, 0x76U,
 1889    0xcaU, 0x82U, 0xc9U, 0x7dU, 0xfaU, 0x59U, 0x47U, 0xf0U,
 1890    0xadU, 0xd4U, 0xa2U, 0xafU, 0x9cU, 0xa4U, 0x72U, 0xc0U,
 1891    0xb7U, 0xfdU, 0x93U, 0x26U, 0x36U, 0x3fU, 0xf7U, 0xccU,
 1892    0x34U, 0xa5U, 0xe5U, 0xf1U, 0x71U, 0xd8U, 0x31U, 0x15U,
 1893    0x04U, 0xc7U, 0x23U, 0xc3U, 0x18U, 0x96U, 0x05U, 0x9aU,
 1894    0x07U, 0x12U, 0x80U, 0xe2U, 0xebU, 0x27U, 0xb2U, 0x75U,
 1895    0x09U, 0x83U, 0x2cU, 0x1aU, 0x1bU, 0x6eU, 0x5aU, 0xa0U,
 1896    0x52U, 0x3bU, 0xd6U, 0xb3U, 0x29U, 0xe3U, 0x2fU, 0x84U,
 1897    0x53U, 0xd1U, 0x00U, 0xedU, 0x20U, 0xfcU, 0xb1U, 0x5bU,
 1898    0x6aU, 0xcbU, 0xbeU, 0x39U, 0x4aU, 0x4cU, 0x58U, 0xcfU,
 1899    0xd0U, 0xefU, 0xaaU, 0xfbU, 0x43U, 0x4dU, 0x33U, 0x85U,
 1900    0x45U, 0xf9U, 0x02U, 0x7fU, 0x50U, 0x3cU, 0x9fU, 0xa8U,
 1901    0x51U, 0xa3U, 0x40U, 0x8fU, 0x92U, 0x9dU, 0x38U, 0xf5U,
 1902    0xbcU, 0xb6U, 0xdaU, 0x21U, 0x10U, 0xffU, 0xf3U, 0xd2U,
 1903    0xcdU, 0x0cU, 0x13U, 0xecU, 0x5fU, 0x97U, 0x44U, 0x17U,
 1904    0xc4U, 0xa7U, 0x7eU, 0x3dU, 0x64U, 0x5dU, 0x19U, 0x73U,
 1905    0x60U, 0x81U, 0x4fU, 0xdcU, 0x22U, 0x2aU, 0x90U, 0x88U,
 1906    0x46U, 0xeeU, 0xb8U, 0x14U, 0xdeU, 0x5eU, 0x0bU, 0xdbU,
 1907    0xe0U, 0x32U, 0x3aU, 0x0aU, 0x49U, 0x06U, 0x24U, 0x5cU,
 1908    0xc2U, 0xd3U, 0xacU, 0x62U, 0x91U, 0x95U, 0xe4U, 0x79U,
 1909    0xe7U, 0xc8U, 0x37U, 0x6dU, 0x8dU, 0xd5U, 0x4eU, 0xa9U,
 1910    0x6cU, 0x56U, 0xf4U, 0xeaU, 0x65U, 0x7aU, 0xaeU, 0x08U,
 1911    0xbaU, 0x78U, 0x25U, 0x2eU, 0x1cU, 0xa6U, 0xb4U, 0xc6U,
 1912    0xe8U, 0xddU, 0x74U, 0x1fU, 0x4bU, 0xbdU, 0x8bU, 0x8aU,
 1913    0x70U, 0x3eU, 0xb5U, 0x66U, 0x48U, 0x03U, 0xf6U, 0x0eU,
 1914    0x61U, 0x35U, 0x57U, 0xb9U, 0x86U, 0xc1U, 0x1dU, 0x9eU,
 1915    0xe1U, 0xf8U, 0x98U, 0x11U, 0x69U, 0xd9U, 0x8eU, 0x94U,
 1916    0x9bU, 0x1eU, 0x87U, 0xe9U, 0xceU, 0x55U, 0x28U, 0xdfU,
 1917    0x8cU, 0xa1U, 0x89U, 0x0dU, 0xbfU, 0xe6U, 0x42U, 0x68U,
 1918    0x41U, 0x99U, 0x2dU, 0x0fU, 0xb0U, 0x54U, 0xbbU, 0x16U
 1919};
 1920
 1921#define AES_XTIME(x)    ((byte)((byte)((x) << 1) ^ ((0 - ((x) >> 7)) & 0x1b)))
 1922
 1923static WARN_UNUSED_RESULT word32 col_mul(
 1924    word32 t, int i2, int i3, int ia, int ib)
 1925{
 1926    byte t3 = GETBYTE(t, i3);
 1927    byte tm = AES_XTIME(GETBYTE(t, i2) ^ t3);
 1928
 1929    return GETBYTE(t, ia) ^ GETBYTE(t, ib) ^ t3 ^ tm;
 1930}
 1931
 1932#if defined(HAVE_AES_DECRYPT) && \
 1933    (defined(HAVE_AES_CBC) || defined(HAVE_AES_ECB) || \
 1934     defined(WOLFSSL_AES_DIRECT))
 1935static WARN_UNUSED_RESULT word32 inv_col_mul(
 1936    word32 t, int i9, int ib, int id, int ie)
 1937{
 1938    byte t9 = GETBYTE(t, i9);
 1939    byte tb = GETBYTE(t, ib);
 1940    byte td = GETBYTE(t, id);
 1941    byte te = GETBYTE(t, ie);
 1942    byte t0 = t9 ^ tb ^ td;
 1943    return t0 ^ AES_XTIME(AES_XTIME(AES_XTIME(t0 ^ te) ^ td ^ te) ^ tb ^ te);
 1944}
 1945#endif /* HAVE_AES_DECRYPT && (HAVE_AES_CBC || HAVE_AES_ECB || WOLFSSL_AES_DIRECT) */
 1946#endif /* WOLFSSL_AES_SMALL_TABLES */
 1947#endif
 1948#endif
 1949
 1950#if defined(HAVE_AES_CBC) || defined(WOLFSSL_AES_DIRECT) || \
 1951                                    defined(HAVE_AESCCM) || defined(HAVE_AESGCM)
 1952#if !defined(WOLFSSL_ARMASM) || defined(WOLFSSL_AES_DIRECT) || \
 1953      defined(HAVE_AESCCM)
 1954
 1955
 1956#ifndef WC_AES_BITSLICED
 1957
 1958#ifndef WC_CACHE_LINE_SZ
 1959    #if defined(__x86_64__) || defined(_M_X64) || \
 1960       (defined(__ILP32__) && (__ILP32__ >= 1))
 1961        #define WC_CACHE_LINE_SZ 64
 1962    #else
 1963        /* default cache line size */
 1964        #define WC_CACHE_LINE_SZ 32
 1965    #endif
 1966#endif
 1967
 1968#ifndef WC_NO_CACHE_RESISTANT
 1969
 1970#if defined(__riscv) && !defined(WOLFSSL_AES_TOUCH_LINES)
 1971    #define WOLFSSL_AES_TOUCH_LINES
 1972#endif
 1973
 1974#ifndef WOLFSSL_AES_SMALL_TABLES
 1975/* load 4 Te Tables into cache by cache line stride */
 1976static WARN_UNUSED_RESULT WC_INLINE word32 PreFetchTe(void)
 1977{
 1978#ifndef WOLFSSL_AES_TOUCH_LINES
 1979    volatile word32 x = 0;
 1980    int i;
 1981    int j;
 1982
 1983    for (i = 0; i < 4; i++) {
 1984        /* 256 elements, each one is 4 bytes */
 1985        for (j = 0; j < 256; j += WC_CACHE_LINE_SZ / 4) {
 1986            x &= Te[i][j];
 1987        }
 1988    }
 1989
 1990    return x;
 1991#else
 1992    return 0;
 1993#endif
 1994}
 1995#else
 1996/* load sbox into cache by cache line stride */
 1997static WARN_UNUSED_RESULT WC_INLINE word32 PreFetchSBox(void)
 1998{
 1999#ifndef WOLFSSL_AES_TOUCH_LINES
 2000    volatile word32 x = 0;
 2001    int i;
 2002
 2003    for (i = 0; i < 256; i += WC_CACHE_LINE_SZ/4) {
 2004        x &= Tsbox[i];
 2005    }
 2006
 2007    return x;
 2008#else
 2009    return 0;
 2010#endif
 2011}
 2012#endif
 2013#endif
 2014
 2015#ifdef WOLFSSL_AES_TOUCH_LINES
 2016#if WC_CACHE_LINE_SZ == 128
 2017    #define WC_CACHE_LINE_BITS      5
 2018    #define WC_CACHE_LINE_MASK_HI   0xe0
 2019    #define WC_CACHE_LINE_MASK_LO   0x1f
 2020    #define WC_CACHE_LINE_ADD       0x20
 2021#elif WC_CACHE_LINE_SZ == 64
 2022    #define WC_CACHE_LINE_BITS      4
 2023    #define WC_CACHE_LINE_MASK_HI   0xf0
 2024    #define WC_CACHE_LINE_MASK_LO   0x0f
 2025    #define WC_CACHE_LINE_ADD       0x10
 2026#elif WC_CACHE_LINE_SZ == 32
 2027    #define WC_CACHE_LINE_BITS      3
 2028    #define WC_CACHE_LINE_MASK_HI   0xf8
 2029    #define WC_CACHE_LINE_MASK_LO   0x07
 2030    #define WC_CACHE_LINE_ADD       0x08
 2031#elif WC_CACHE_LINE_SZ == 16
 2032    #define WC_CACHE_LINE_BITS      2
 2033    #define WC_CACHE_LINE_MASK_HI   0xfc
 2034    #define WC_CACHE_LINE_MASK_LO   0x03
 2035    #define WC_CACHE_LINE_ADD       0x04
 2036#else
 2037    #error Cache line size not supported
 2038#endif
 2039
 2040#ifndef WOLFSSL_AES_SMALL_TABLES
 2041static word32 GetTable(const word32* t, byte o)
 2042{
 2043#if WC_CACHE_LINE_SZ == 64
 2044    word32 e;
 2045    byte hi = o & 0xf0;
 2046    byte lo = o & 0x0f;
 2047
 2048    e  = t[lo + 0x00] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2049    e |= t[lo + 0x10] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2050    e |= t[lo + 0x20] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2051    e |= t[lo + 0x30] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2052    e |= t[lo + 0x40] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2053    e |= t[lo + 0x50] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2054    e |= t[lo + 0x60] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2055    e |= t[lo + 0x70] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2056    e |= t[lo + 0x80] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2057    e |= t[lo + 0x90] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2058    e |= t[lo + 0xa0] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2059    e |= t[lo + 0xb0] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2060    e |= t[lo + 0xc0] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2061    e |= t[lo + 0xd0] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2062    e |= t[lo + 0xe0] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2063    e |= t[lo + 0xf0] & ((word32)0 - (((word32)hi - 0x01) >> 31));
 2064
 2065    return e;
 2066#else
 2067    word32 e = 0;
 2068    int i;
 2069    byte hi = o & WC_CACHE_LINE_MASK_HI;
 2070    byte lo = o & WC_CACHE_LINE_MASK_LO;
 2071
 2072    for (i = 0; i < 256; i += (1 << WC_CACHE_LINE_BITS)) {
 2073        e |= t[lo + i] & ((word32)0 - (((word32)hi - 0x01) >> 31));
 2074        hi -= WC_CACHE_LINE_ADD;
 2075    }
 2076
 2077    return e;
 2078#endif
 2079}
 2080#endif
 2081
 2082#ifdef WOLFSSL_AES_SMALL_TABLES
 2083static byte GetTable8(const byte* t, byte o)
 2084{
 2085#if WC_CACHE_LINE_SZ == 64
 2086    byte e;
 2087    byte hi = o & 0xf0;
 2088    byte lo = o & 0x0f;
 2089
 2090    e  = t[lo + 0x00] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2091    e |= t[lo + 0x10] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2092    e |= t[lo + 0x20] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2093    e |= t[lo + 0x30] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2094    e |= t[lo + 0x40] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2095    e |= t[lo + 0x50] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2096    e |= t[lo + 0x60] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2097    e |= t[lo + 0x70] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2098    e |= t[lo + 0x80] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2099    e |= t[lo + 0x90] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2100    e |= t[lo + 0xa0] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2101    e |= t[lo + 0xb0] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2102    e |= t[lo + 0xc0] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2103    e |= t[lo + 0xd0] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2104    e |= t[lo + 0xe0] & ((word32)0 - (((word32)hi - 0x01) >> 31)); hi -= 0x10;
 2105    e |= t[lo + 0xf0] & ((word32)0 - (((word32)hi - 0x01) >> 31));
 2106
 2107    return e;
 2108#else
 2109    byte e = 0;
 2110    int i;
 2111    byte hi = o & WC_CACHE_LINE_MASK_HI;
 2112    byte lo = o & WC_CACHE_LINE_MASK_LO;
 2113
 2114    for (i = 0; i < 256; i += (1 << WC_CACHE_LINE_BITS)) {
 2115        e |= t[lo + i] & ((word32)0 - (((word32)hi - 0x01) >> 31));
 2116        hi -= WC_CACHE_LINE_ADD;
 2117    }
 2118
 2119    return e;
 2120#endif
 2121}
 2122#endif
 2123
 2124#ifndef WOLFSSL_AES_SMALL_TABLES
 2125static void GetTable_Multi(const word32* t, word32* t0, byte o0,
 2126    word32* t1, byte o1, word32* t2, byte o2, word32* t3, byte o3)
 2127{
 2128    word32 e0 = 0;
 2129    word32 e1 = 0;
 2130    word32 e2 = 0;
 2131    word32 e3 = 0;
 2132    byte hi0 = o0 & WC_CACHE_LINE_MASK_HI;
 2133    byte lo0 = o0 & WC_CACHE_LINE_MASK_LO;
 2134    byte hi1 = o1 & WC_CACHE_LINE_MASK_HI;
 2135    byte lo1 = o1 & WC_CACHE_LINE_MASK_LO;
 2136    byte hi2 = o2 & WC_CACHE_LINE_MASK_HI;
 2137    byte lo2 = o2 & WC_CACHE_LINE_MASK_LO;
 2138    byte hi3 = o3 & WC_CACHE_LINE_MASK_HI;
 2139    byte lo3 = o3 & WC_CACHE_LINE_MASK_LO;
 2140    int i;
 2141
 2142    for (i = 0; i < 256; i += (1 << WC_CACHE_LINE_BITS)) {
 2143        e0 |= t[lo0 + i] & ((word32)0 - (((word32)hi0 - 0x01) >> 31));
 2144        hi0 -= WC_CACHE_LINE_ADD;
 2145        e1 |= t[lo1 + i] & ((word32)0 - (((word32)hi1 - 0x01) >> 31));
 2146        hi1 -= WC_CACHE_LINE_ADD;
 2147        e2 |= t[lo2 + i] & ((word32)0 - (((word32)hi2 - 0x01) >> 31));
 2148        hi2 -= WC_CACHE_LINE_ADD;
 2149        e3 |= t[lo3 + i] & ((word32)0 - (((word32)hi3 - 0x01) >> 31));
 2150        hi3 -= WC_CACHE_LINE_ADD;
 2151    }
 2152    *t0 = e0;
 2153    *t1 = e1;
 2154    *t2 = e2;
 2155    *t3 = e3;
 2156}
 2157static void XorTable_Multi(const word32* t, word32* t0, byte o0,
 2158    word32* t1, byte o1, word32* t2, byte o2, word32* t3, byte o3)
 2159{
 2160    word32 e0 = 0;
 2161    word32 e1 = 0;
 2162    word32 e2 = 0;
 2163    word32 e3 = 0;
 2164    byte hi0 = o0 & WC_CACHE_LINE_MASK_HI;
 2165    byte lo0 = o0 & WC_CACHE_LINE_MASK_LO;
 2166    byte hi1 = o1 & WC_CACHE_LINE_MASK_HI;
 2167    byte lo1 = o1 & WC_CACHE_LINE_MASK_LO;
 2168    byte hi2 = o2 & WC_CACHE_LINE_MASK_HI;
 2169    byte lo2 = o2 & WC_CACHE_LINE_MASK_LO;
 2170    byte hi3 = o3 & WC_CACHE_LINE_MASK_HI;
 2171    byte lo3 = o3 & WC_CACHE_LINE_MASK_LO;
 2172    int i;
 2173
 2174    for (i = 0; i < 256; i += (1 << WC_CACHE_LINE_BITS)) {
 2175        e0 |= t[lo0 + i] & ((word32)0 - (((word32)hi0 - 0x01) >> 31));
 2176        hi0 -= WC_CACHE_LINE_ADD;
 2177        e1 |= t[lo1 + i] & ((word32)0 - (((word32)hi1 - 0x01) >> 31));
 2178        hi1 -= WC_CACHE_LINE_ADD;
 2179        e2 |= t[lo2 + i] & ((word32)0 - (((word32)hi2 - 0x01) >> 31));
 2180        hi2 -= WC_CACHE_LINE_ADD;
 2181        e3 |= t[lo3 + i] & ((word32)0 - (((word32)hi3 - 0x01) >> 31));
 2182        hi3 -= WC_CACHE_LINE_ADD;
 2183    }
 2184    *t0 ^= e0;
 2185    *t1 ^= e1;
 2186    *t2 ^= e2;
 2187    *t3 ^= e3;
 2188}
 2189static word32 GetTable8_4(const byte* t, byte o0, byte o1, byte o2, byte o3)
 2190{
 2191    word32 e = 0;
 2192    int i;
 2193    byte hi0 = o0 & WC_CACHE_LINE_MASK_HI;
 2194    byte lo0 = o0 & WC_CACHE_LINE_MASK_LO;
 2195    byte hi1 = o1 & WC_CACHE_LINE_MASK_HI;
 2196    byte lo1 = o1 & WC_CACHE_LINE_MASK_LO;
 2197    byte hi2 = o2 & WC_CACHE_LINE_MASK_HI;
 2198    byte lo2 = o2 & WC_CACHE_LINE_MASK_LO;
 2199    byte hi3 = o3 & WC_CACHE_LINE_MASK_HI;
 2200    byte lo3 = o3 & WC_CACHE_LINE_MASK_LO;
 2201
 2202    for (i = 0; i < 256; i += (1 << WC_CACHE_LINE_BITS)) {
 2203        e |= (word32)(t[lo0 + i] & ((word32)0 - (((word32)hi0 - 0x01) >> 31)))
 2204             << 24;
 2205        hi0 -= WC_CACHE_LINE_ADD;
 2206        e |= (word32)(t[lo1 + i] & ((word32)0 - (((word32)hi1 - 0x01) >> 31)))
 2207             << 16;
 2208        hi1 -= WC_CACHE_LINE_ADD;
 2209        e |= (word32)(t[lo2 + i] & ((word32)0 - (((word32)hi2 - 0x01) >> 31)))
 2210             <<  8;
 2211        hi2 -= WC_CACHE_LINE_ADD;
 2212        e |= (word32)(t[lo3 + i] & ((word32)0 - (((word32)hi3 - 0x01) >> 31)))
 2213             <<  0;
 2214        hi3 -= WC_CACHE_LINE_ADD;
 2215    }
 2216
 2217    return e;
 2218}
 2219#endif
 2220#else
 2221
 2222#define GetTable(t, o)  t[o]
 2223#define GetTable8(t, o) t[o]
 2224#define GetTable_Multi(t, t0, o0, t1, o1, t2, o2, t3, o3)  \
 2225    *(t0) = (t)[o0]; *(t1) = (t)[o1]; *(t2) = (t)[o2]; *(t3) = (t)[o3]
 2226#define XorTable_Multi(t, t0, o0, t1, o1, t2, o2, t3, o3)  \
 2227    *(t0) ^= (t)[o0]; *(t1) ^= (t)[o1]; *(t2) ^= (t)[o2]; *(t3) ^= (t)[o3]
 2228#define GetTable8_4(t, o0, o1, o2, o3) \
 2229    (((word32)(t)[o0] << 24) | ((word32)(t)[o1] << 16) |   \
 2230     ((word32)(t)[o2] <<  8) | ((word32)(t)[o3] <<  0))
 2231#endif
 2232
 2233#ifndef HAVE_CUDA
 2234/* Encrypt a block using AES.
 2235 *
 2236 * @param [in]  aes       AES object.
 2237 * @param [in]  inBlock   Block to encrypt.
 2238 * @param [out] outBlock  Encrypted block.
 2239 * @param [in]  r         Rounds divided by 2.
 2240 */
 2241#define WC_AES_HAVE_PREFETCH_ARG
 2242static int always_prefetch = 0;
 2243WC_MAYBE_UNUSED static int never_prefetch = 1;
 2244WC_ARGS_NOT_NULL((1, 2, 3, 5))
 2245static void AesEncrypt_C(Aes* aes, const byte* inBlock, byte* outBlock,
 2246        word32 r, int *prefetch_ptr)
 2247{
 2248    word32 s0 = 0, s1 = 0, s2 = 0, s3 = 0;
 2249    word32 t0 = 0, t1 = 0, t2 = 0, t3 = 0;
 2250    const word32* rk;
 2251
 2252#ifdef WC_C_DYNAMIC_FALLBACK
 2253    rk = aes->key_C_fallback;
 2254#else
 2255    rk = aes->key;
 2256#endif
 2257
 2258    /*
 2259     * map byte array block to cipher state
 2260     * and add initial round key:
 2261     */
 2262    XMEMCPY(&s0, inBlock,                  sizeof(s0));
 2263    XMEMCPY(&s1, inBlock +     sizeof(s0), sizeof(s1));
 2264    XMEMCPY(&s2, inBlock + 2 * sizeof(s0), sizeof(s2));
 2265    XMEMCPY(&s3, inBlock + 3 * sizeof(s0), sizeof(s3));
 2266
 2267#ifdef LITTLE_ENDIAN_ORDER
 2268    s0 = ByteReverseWord32(s0);
 2269    s1 = ByteReverseWord32(s1);
 2270    s2 = ByteReverseWord32(s2);
 2271    s3 = ByteReverseWord32(s3);
 2272#endif
 2273
 2274    /* AddRoundKey */
 2275    s0 ^= rk[0];
 2276    s1 ^= rk[1];
 2277    s2 ^= rk[2];
 2278    s3 ^= rk[3];
 2279
 2280#ifndef WOLFSSL_AES_SMALL_TABLES
 2281
 2282#ifndef WC_NO_CACHE_RESISTANT
 2283    if (*prefetch_ptr == 0) {
 2284        s0 |= PreFetchTe();
 2285        if (prefetch_ptr != &always_prefetch)
 2286            *prefetch_ptr = 1;
 2287    }
 2288#else
 2289    (void)prefetch_ptr;
 2290#endif
 2291
 2292#ifndef WOLFSSL_AES_TOUCH_LINES
 2293#define ENC_ROUND_T_S(o)                                                       \
 2294    t0 = GetTable(Te[0], GETBYTE(s0, 3)) ^ GetTable(Te[1], GETBYTE(s1, 2)) ^   \
 2295         GetTable(Te[2], GETBYTE(s2, 1)) ^ GetTable(Te[3], GETBYTE(s3, 0)) ^   \
 2296         rk[(o)+4];                                                            \
 2297    t1 = GetTable(Te[0], GETBYTE(s1, 3)) ^ GetTable(Te[1], GETBYTE(s2, 2)) ^   \
 2298         GetTable(Te[2], GETBYTE(s3, 1)) ^ GetTable(Te[3], GETBYTE(s0, 0)) ^   \
 2299         rk[(o)+5];                                                            \
 2300    t2 = GetTable(Te[0], GETBYTE(s2, 3)) ^ GetTable(Te[1], GETBYTE(s3, 2)) ^   \
 2301         GetTable(Te[2], GETBYTE(s0, 1)) ^ GetTable(Te[3], GETBYTE(s1, 0)) ^   \
 2302         rk[(o)+6];                                                            \
 2303    t3 = GetTable(Te[0], GETBYTE(s3, 3)) ^ GetTable(Te[1], GETBYTE(s0, 2)) ^   \
 2304         GetTable(Te[2], GETBYTE(s1, 1)) ^ GetTable(Te[3], GETBYTE(s2, 0)) ^   \
 2305         rk[(o)+7]
 2306#define ENC_ROUND_S_T(o)                                                       \
 2307    s0 = GetTable(Te[0], GETBYTE(t0, 3)) ^ GetTable(Te[1], GETBYTE(t1, 2)) ^   \
 2308         GetTable(Te[2], GETBYTE(t2, 1)) ^ GetTable(Te[3], GETBYTE(t3, 0)) ^   \
 2309         rk[(o)+0];                                                            \
 2310    s1 = GetTable(Te[0], GETBYTE(t1, 3)) ^ GetTable(Te[1], GETBYTE(t2, 2)) ^   \
 2311         GetTable(Te[2], GETBYTE(t3, 1)) ^ GetTable(Te[3], GETBYTE(t0, 0)) ^   \
 2312         rk[(o)+1];                                                            \
 2313    s2 = GetTable(Te[0], GETBYTE(t2, 3)) ^ GetTable(Te[1], GETBYTE(t3, 2)) ^   \
 2314         GetTable(Te[2], GETBYTE(t0, 1)) ^ GetTable(Te[3], GETBYTE(t1, 0)) ^   \
 2315         rk[(o)+2];                                                            \
 2316    s3 = GetTable(Te[0], GETBYTE(t3, 3)) ^ GetTable(Te[1], GETBYTE(t0, 2)) ^   \
 2317         GetTable(Te[2], GETBYTE(t1, 1)) ^ GetTable(Te[3], GETBYTE(t2, 0)) ^   \
 2318         rk[(o)+3]
 2319#else
 2320#define ENC_ROUND_T_S(o)                                                       \
 2321    GetTable_Multi(Te[0], &t0, GETBYTE(s0, 3), &t1, GETBYTE(s1, 3),            \
 2322                          &t2, GETBYTE(s2, 3), &t3, GETBYTE(s3, 3));           \
 2323    XorTable_Multi(Te[1], &t0, GETBYTE(s1, 2), &t1, GETBYTE(s2, 2),            \
 2324                          &t2, GETBYTE(s3, 2), &t3, GETBYTE(s0, 2));           \
 2325    XorTable_Multi(Te[2], &t0, GETBYTE(s2, 1), &t1, GETBYTE(s3, 1),            \
 2326                          &t2, GETBYTE(s0, 1), &t3, GETBYTE(s1, 1));           \
 2327    XorTable_Multi(Te[3], &t0, GETBYTE(s3, 0), &t1, GETBYTE(s0, 0),            \
 2328                          &t2, GETBYTE(s1, 0), &t3, GETBYTE(s2, 0));           \
 2329    t0 ^= rk[(o)+4]; t1 ^= rk[(o)+5]; t2 ^= rk[(o)+6]; t3 ^= rk[(o)+7];
 2330
 2331#define ENC_ROUND_S_T(o)                                                       \
 2332    GetTable_Multi(Te[0], &s0, GETBYTE(t0, 3), &s1, GETBYTE(t1, 3),            \
 2333                          &s2, GETBYTE(t2, 3), &s3, GETBYTE(t3, 3));           \
 2334    XorTable_Multi(Te[1], &s0, GETBYTE(t1, 2), &s1, GETBYTE(t2, 2),            \
 2335                          &s2, GETBYTE(t3, 2), &s3, GETBYTE(t0, 2));           \
 2336    XorTable_Multi(Te[2], &s0, GETBYTE(t2, 1), &s1, GETBYTE(t3, 1),            \
 2337                          &s2, GETBYTE(t0, 1), &s3, GETBYTE(t1, 1));           \
 2338    XorTable_Multi(Te[3], &s0, GETBYTE(t3, 0), &s1, GETBYTE(t0, 0),            \
 2339                          &s2, GETBYTE(t1, 0), &s3, GETBYTE(t2, 0));           \
 2340    s0 ^= rk[(o)+0]; s1 ^= rk[(o)+1]; s2 ^= rk[(o)+2]; s3 ^= rk[(o)+3];
 2341#endif
 2342
 2343#ifndef WOLFSSL_AES_NO_UNROLL
 2344/* Unroll the loop. */
 2345                       ENC_ROUND_T_S( 0);
 2346    ENC_ROUND_S_T( 8); ENC_ROUND_T_S( 8);
 2347    ENC_ROUND_S_T(16); ENC_ROUND_T_S(16);
 2348    ENC_ROUND_S_T(24); ENC_ROUND_T_S(24);
 2349    ENC_ROUND_S_T(32); ENC_ROUND_T_S(32);
 2350    if (r > 5) {
 2351        ENC_ROUND_S_T(40); ENC_ROUND_T_S(40);
 2352        if (r > 6) {
 2353            ENC_ROUND_S_T(48); ENC_ROUND_T_S(48);
 2354        }
 2355    }
 2356    rk += r * 8;
 2357#else
 2358    /*
 2359     * Nr - 1 full rounds:
 2360     */
 2361
 2362    for (;;) {
 2363        ENC_ROUND_T_S(0);
 2364
 2365        rk += 8;
 2366        if (--r == 0) {
 2367            break;
 2368        }
 2369
 2370        ENC_ROUND_S_T(0);
 2371    }
 2372#endif
 2373
 2374    /*
 2375     * apply last round and
 2376     * map cipher state to byte array block:
 2377     */
 2378
 2379#ifndef WOLFSSL_AES_TOUCH_LINES
 2380    s0 =
 2381        (GetTable(Te[2], GETBYTE(t0, 3)) & 0xff000000) ^
 2382        (GetTable(Te[3], GETBYTE(t1, 2)) & 0x00ff0000) ^
 2383        (GetTable(Te[0], GETBYTE(t2, 1)) & 0x0000ff00) ^
 2384        (GetTable(Te[1], GETBYTE(t3, 0)) & 0x000000ff) ^
 2385        rk[0];
 2386    s1 =
 2387        (GetTable(Te[2], GETBYTE(t1, 3)) & 0xff000000) ^
 2388        (GetTable(Te[3], GETBYTE(t2, 2)) & 0x00ff0000) ^
 2389        (GetTable(Te[0], GETBYTE(t3, 1)) & 0x0000ff00) ^
 2390        (GetTable(Te[1], GETBYTE(t0, 0)) & 0x000000ff) ^
 2391        rk[1];
 2392    s2 =
 2393        (GetTable(Te[2], GETBYTE(t2, 3)) & 0xff000000) ^
 2394        (GetTable(Te[3], GETBYTE(t3, 2)) & 0x00ff0000) ^
 2395        (GetTable(Te[0], GETBYTE(t0, 1)) & 0x0000ff00) ^
 2396        (GetTable(Te[1], GETBYTE(t1, 0)) & 0x000000ff) ^
 2397        rk[2];
 2398    s3 =
 2399        (GetTable(Te[2], GETBYTE(t3, 3)) & 0xff000000) ^
 2400        (GetTable(Te[3], GETBYTE(t0, 2)) & 0x00ff0000) ^
 2401        (GetTable(Te[0], GETBYTE(t1, 1)) & 0x0000ff00) ^
 2402        (GetTable(Te[1], GETBYTE(t2, 0)) & 0x000000ff) ^
 2403        rk[3];
 2404#else
 2405{
 2406    word32 u0;
 2407    word32 u1;
 2408    word32 u2;
 2409    word32 u3;
 2410
 2411    s0 = rk[0]; s1 = rk[1]; s2 = rk[2]; s3 = rk[3];
 2412    GetTable_Multi(Te[2], &u0, GETBYTE(t0, 3), &u1, GETBYTE(t1, 3),
 2413                          &u2, GETBYTE(t2, 3), &u3, GETBYTE(t3, 3));
 2414    s0 ^= u0 & 0xff000000; s1 ^= u1 & 0xff000000;
 2415    s2 ^= u2 & 0xff000000; s3 ^= u3 & 0xff000000;
 2416    GetTable_Multi(Te[3], &u0, GETBYTE(t1, 2), &u1, GETBYTE(t2, 2),
 2417                          &u2, GETBYTE(t3, 2), &u3, GETBYTE(t0, 2));
 2418    s0 ^= u0 & 0x00ff0000; s1 ^= u1 & 0x00ff0000;
 2419    s2 ^= u2 & 0x00ff0000; s3 ^= u3 & 0x00ff0000;
 2420    GetTable_Multi(Te[0], &u0, GETBYTE(t2, 1), &u1, GETBYTE(t3, 1),
 2421                          &u2, GETBYTE(t0, 1), &u3, GETBYTE(t1, 1));
 2422    s0 ^= u0 & 0x0000ff00; s1 ^= u1 & 0x0000ff00;
 2423    s2 ^= u2 & 0x0000ff00; s3 ^= u3 & 0x0000ff00;
 2424    GetTable_Multi(Te[1], &u0, GETBYTE(t3, 0), &u1, GETBYTE(t0, 0),
 2425                          &u2, GETBYTE(t1, 0), &u3, GETBYTE(t2, 0));
 2426    s0 ^= u0 & 0x000000ff; s1 ^= u1 & 0x000000ff;
 2427    s2 ^= u2 & 0x000000ff; s3 ^= u3 & 0x000000ff;
 2428}
 2429#endif
 2430
 2431#else /* WOLFSSL_AES_SMALL_TABLES */
 2432
 2433#ifndef WC_NO_CACHE_RESISTANT
 2434    if (*prefetch_ptr == 0) {
 2435        s0 |= PreFetchSBox();
 2436        if (prefetch_ptr != &always_prefetch)
 2437            *prefetch_ptr = 1;
 2438    }
 2439#else
 2440    (void)prefetch_ptr;
 2441#endif
 2442
 2443    r *= 2;
 2444    /* Two rounds at a time */
 2445    for (rk += 4; r > 1; r--, rk += 4) {
 2446        t0 =
 2447            ((word32)GetTable8(Tsbox, GETBYTE(s0, 3)) << 24) ^
 2448            ((word32)GetTable8(Tsbox, GETBYTE(s1, 2)) << 16) ^
 2449            ((word32)GetTable8(Tsbox, GETBYTE(s2, 1)) <<  8) ^
 2450            ((word32)GetTable8(Tsbox, GETBYTE(s3, 0)));
 2451        t1 =
 2452            ((word32)GetTable8(Tsbox, GETBYTE(s1, 3)) << 24) ^
 2453            ((word32)GetTable8(Tsbox, GETBYTE(s2, 2)) << 16) ^
 2454            ((word32)GetTable8(Tsbox, GETBYTE(s3, 1)) <<  8) ^
 2455            ((word32)GetTable8(Tsbox, GETBYTE(s0, 0)));
 2456        t2 =
 2457            ((word32)GetTable8(Tsbox, GETBYTE(s2, 3)) << 24) ^
 2458            ((word32)GetTable8(Tsbox, GETBYTE(s3, 2)) << 16) ^
 2459            ((word32)GetTable8(Tsbox, GETBYTE(s0, 1)) <<  8) ^
 2460            ((word32)GetTable8(Tsbox, GETBYTE(s1, 0)));
 2461        t3 =
 2462            ((word32)GetTable8(Tsbox, GETBYTE(s3, 3)) << 24) ^
 2463            ((word32)GetTable8(Tsbox, GETBYTE(s0, 2)) << 16) ^
 2464            ((word32)GetTable8(Tsbox, GETBYTE(s1, 1)) <<  8) ^
 2465            ((word32)GetTable8(Tsbox, GETBYTE(s2, 0)));
 2466
 2467        s0 =
 2468            (col_mul(t0, 3, 2, 0, 1) << 24) ^
 2469            (col_mul(t0, 2, 1, 0, 3) << 16) ^
 2470            (col_mul(t0, 1, 0, 2, 3) <<  8) ^
 2471            (col_mul(t0, 0, 3, 2, 1)      ) ^
 2472            rk[0];
 2473        s1 =
 2474            (col_mul(t1, 3, 2, 0, 1) << 24) ^
 2475            (col_mul(t1, 2, 1, 0, 3) << 16) ^
 2476            (col_mul(t1, 1, 0, 2, 3) <<  8) ^
 2477            (col_mul(t1, 0, 3, 2, 1)      ) ^
 2478            rk[1];
 2479        s2 =
 2480            (col_mul(t2, 3, 2, 0, 1) << 24) ^
 2481            (col_mul(t2, 2, 1, 0, 3) << 16) ^
 2482            (col_mul(t2, 1, 0, 2, 3) <<  8) ^
 2483            (col_mul(t2, 0, 3, 2, 1)      ) ^
 2484            rk[2];
 2485        s3 =
 2486            (col_mul(t3, 3, 2, 0, 1) << 24) ^
 2487            (col_mul(t3, 2, 1, 0, 3) << 16) ^
 2488            (col_mul(t3, 1, 0, 2, 3) <<  8) ^
 2489            (col_mul(t3, 0, 3, 2, 1)      ) ^
 2490            rk[3];
 2491    }
 2492
 2493    t0 =
 2494        ((word32)GetTable8(Tsbox, GETBYTE(s0, 3)) << 24) ^
 2495        ((word32)GetTable8(Tsbox, GETBYTE(s1, 2)) << 16) ^
 2496        ((word32)GetTable8(Tsbox, GETBYTE(s2, 1)) <<  8) ^
 2497        ((word32)GetTable8(Tsbox, GETBYTE(s3, 0)));
 2498    t1 =
 2499        ((word32)GetTable8(Tsbox, GETBYTE(s1, 3)) << 24) ^
 2500        ((word32)GetTable8(Tsbox, GETBYTE(s2, 2)) << 16) ^
 2501        ((word32)GetTable8(Tsbox, GETBYTE(s3, 1)) <<  8) ^
 2502        ((word32)GetTable8(Tsbox, GETBYTE(s0, 0)));
 2503    t2 =
 2504        ((word32)GetTable8(Tsbox, GETBYTE(s2, 3)) << 24) ^
 2505        ((word32)GetTable8(Tsbox, GETBYTE(s3, 2)) << 16) ^
 2506        ((word32)GetTable8(Tsbox, GETBYTE(s0, 1)) <<  8) ^
 2507        ((word32)GetTable8(Tsbox, GETBYTE(s1, 0)));
 2508    t3 =
 2509        ((word32)GetTable8(Tsbox, GETBYTE(s3, 3)) << 24) ^
 2510        ((word32)GetTable8(Tsbox, GETBYTE(s0, 2)) << 16) ^
 2511        ((word32)GetTable8(Tsbox, GETBYTE(s1, 1)) <<  8) ^
 2512        ((word32)GetTable8(Tsbox, GETBYTE(s2, 0)));
 2513    s0 = t0 ^ rk[0];
 2514    s1 = t1 ^ rk[1];
 2515    s2 = t2 ^ rk[2];
 2516    s3 = t3 ^ rk[3];
 2517
 2518#endif /* WOLFSSL_AES_SMALL_TABLES */
 2519
 2520    /* write out */
 2521#ifdef LITTLE_ENDIAN_ORDER
 2522    s0 = ByteReverseWord32(s0);
 2523    s1 = ByteReverseWord32(s1);
 2524    s2 = ByteReverseWord32(s2);
 2525    s3 = ByteReverseWord32(s3);
 2526#endif
 2527
 2528    XMEMCPY(outBlock,                  &s0, sizeof(s0));
 2529    XMEMCPY(outBlock +     sizeof(s0), &s1, sizeof(s1));
 2530    XMEMCPY(outBlock + 2 * sizeof(s0), &s2, sizeof(s2));
 2531    XMEMCPY(outBlock + 3 * sizeof(s0), &s3, sizeof(s3));
 2532}
 2533
 2534#if defined(HAVE_AES_ECB) && !(defined(WOLFSSL_IMX6_CAAM) && \
 2535    !defined(NO_IMX6_CAAM_AES) && !defined(WOLFSSL_QNX_CAAM)) && \
 2536    !defined(MAX3266X_AES)
 2537#if !defined(WOLFSSL_ARMASM) || defined(__aarch64__)
 2538/* Encrypt a number of blocks using AES.
 2539 *
 2540 * @param [in]  aes  AES object.
 2541 * @param [in]  in   Block to encrypt.
 2542 * @param [out] out  Encrypted block.
 2543 * @param [in]  sz   Number of blocks to encrypt.
 2544 */
 2545static void AesEncryptBlocks_C(Aes* aes, const byte* in, byte* out, word32 sz)
 2546{
 2547    word32 i;
 2548    int did_prefetches = 0;
 2549
 2550    for (i = 0; i < sz; i += WC_AES_BLOCK_SIZE) {
 2551        AesEncrypt_C(aes, in, out, aes->rounds >> 1, &did_prefetches);
 2552        in += WC_AES_BLOCK_SIZE;
 2553        out += WC_AES_BLOCK_SIZE;
 2554    }
 2555}
 2556#endif
 2557#endif
 2558#else
 2559extern void AesEncrypt_C(Aes* aes, const byte* inBlock, byte* outBlock,
 2560        word32 r);
 2561extern void AesEncryptBlocks_C(Aes* aes, const byte* in, byte* out, word32 sz);
 2562#endif /* HAVE_CUDA */
 2563
 2564#else
 2565
 2566/* Bit-sliced implementation based on work by "circuit minimization team" (CMT):
 2567 *   http://cs-www.cs.yale.edu/homes/peralta/CircuitStuff/CMT.html
 2568 */
 2569/* http://cs-www.cs.yale.edu/homes/peralta/CircuitStuff/SLP_AES_113.txt */
 2570static void bs_sub_bytes(bs_word u[8])
 2571{
 2572    bs_word y1, y2, y3, y4, y5, y6, y7, y8, y9;
 2573    bs_word y10, y11, y12, y13, y14, y15, y16, y17, y18, y19;
 2574    bs_word y20, y21;
 2575    bs_word t0, t1, t2, t3, t4, t5, t6, t7, t8, t9;
 2576    bs_word t10, t11, t12, t13, t14, t15, t16, t17, t18, t19;
 2577    bs_word t20, t21, t22, t23, t24, t25, t26, t27, t28, t29;
 2578    bs_word t30, t31, t32, t33, t34, t35, t36, t37, t38, t39;
 2579    bs_word t40, t41, t42, t43, t44, t45;
 2580    bs_word z0, z1, z2, z3, z4, z5, z6, z7, z8, z9;
 2581    bs_word z10, z11, z12, z13, z14, z15, z16, z17;
 2582    bs_word tc1, tc2, tc3, tc4, tc5, tc6, tc7, tc8, tc9;
 2583    bs_word tc10, tc11, tc12, tc13, tc14, tc16, tc17, tc18;
 2584    bs_word tc20, tc21, tc26;
 2585    bs_word U0, U1, U2, U3, U4, U5, U6, U7;
 2586    bs_word S0, S1, S2, S3, S4, S5, S6, S7;
 2587
 2588    U0 = u[7];
 2589    U1 = u[6];
 2590    U2 = u[5];
 2591    U3 = u[4];
 2592    U4 = u[3];
 2593    U5 = u[2];
 2594    U6 = u[1];
 2595    U7 = u[0];
 2596
 2597    y14 = U3 ^ U5;
 2598    y13 = U0 ^ U6;
 2599    y9 = U0 ^ U3;
 2600    y8 = U0 ^ U5;
 2601    t0 = U1 ^ U2;
 2602    y1 = t0 ^ U7;
 2603    y4 = y1 ^ U3;
 2604    y12 = y13 ^ y14;
 2605    y2 = y1 ^ U0;
 2606    y5 = y1 ^ U6;
 2607    y3 = y5 ^ y8;
 2608    t1 = U4 ^ y12;
 2609    y15 = t1 ^ U5;
 2610    y20 = t1 ^ U1;
 2611    y6 = y15 ^ U7;
 2612    y10 = y15 ^ t0;
 2613    y11 = y20 ^ y9;
 2614    y7 = U7 ^ y11;
 2615    y17 = y10 ^ y11;
 2616    y19 = y10 ^ y8;
 2617    y16 = t0 ^ y11;
 2618    y21 = y13 ^ y16;
 2619    y18 = U0 ^ y16;
 2620    t2 = y12 & y15;
 2621    t3 = y3 & y6;
 2622    t4 = t3 ^ t2;
 2623    t5 = y4 & U7;
 2624    t6 = t5 ^ t2;
 2625    t7 = y13 & y16;
 2626    t8 = y5 & y1;
 2627    t9 = t8 ^ t7;
 2628    t10 = y2 & y7;
 2629    t11 = t10 ^ t7;
 2630    t12 = y9 & y11;
 2631    t13 = y14 & y17;
 2632    t14 = t13 ^ t12;
 2633    t15 = y8 & y10;
 2634    t16 = t15 ^ t12;
 2635    t17 = t4 ^ y20;
 2636    t18 = t6 ^ t16;
 2637    t19 = t9 ^ t14;
 2638    t20 = t11 ^ t16;
 2639    t21 = t17 ^ t14;
 2640    t22 = t18 ^ y19;
 2641    t23 = t19 ^ y21;
 2642    t24 = t20 ^ y18;
 2643    t25 = t21 ^ t22;
 2644    t26 = t21 & t23;
 2645    t27 = t24 ^ t26;
 2646    t28 = t25 & t27;
 2647    t29 = t28 ^ t22;
 2648    t30 = t23 ^ t24;
 2649    t31 = t22 ^ t26;
 2650    t32 = t31 & t30;
 2651    t33 = t32 ^ t24;
 2652    t34 = t23 ^ t33;
 2653    t35 = t27 ^ t33;
 2654    t36 = t24 & t35;
 2655    t37 = t36 ^ t34;
 2656    t38 = t27 ^ t36;
 2657    t39 = t29 & t38;
 2658    t40 = t25 ^ t39;
 2659    t41 = t40 ^ t37;
 2660    t42 = t29 ^ t33;
 2661    t43 = t29 ^ t40;
 2662    t44 = t33 ^ t37;
 2663    t45 = t42 ^ t41;
 2664    z0 = t44 & y15;
 2665    z1 = t37 & y6;
 2666    z2 = t33 & U7;
 2667    z3 = t43 & y16;
 2668    z4 = t40 & y1;
 2669    z5 = t29 & y7;
 2670    z6 = t42 & y11;
 2671    z7 = t45 & y17;
 2672    z8 = t41 & y10;
 2673    z9 = t44 & y12;
 2674    z10 = t37 & y3;
 2675    z11 = t33 & y4;
 2676    z12 = t43 & y13;
 2677    z13 = t40 & y5;
 2678    z14 = t29 & y2;
 2679    z15 = t42 & y9;
 2680    z16 = t45 & y14;
 2681    z17 = t41 & y8;
 2682    tc1 = z15 ^ z16;
 2683    tc2 = z10 ^ tc1;
 2684    tc3 = z9 ^ tc2;
 2685    tc4 = z0 ^ z2;
 2686    tc5 = z1 ^ z0;
 2687    tc6 = z3 ^ z4;
 2688    tc7 = z12 ^ tc4;
 2689    tc8 = z7 ^ tc6;
 2690    tc9 = z8 ^ tc7;
 2691    tc10 = tc8 ^ tc9;
 2692    tc11 = tc6 ^ tc5;
 2693    tc12 = z3 ^ z5;
 2694    tc13 = z13 ^ tc1;
 2695    tc14 = tc4 ^ tc12;
 2696    S3 = tc3 ^ tc11;
 2697    tc16 = z6 ^ tc8;
 2698    tc17 = z14 ^ tc10;
 2699    tc18 = tc13 ^ tc14;
 2700    S7 = ~(z12 ^ tc18);
 2701    tc20 = z15 ^ tc16;
 2702    tc21 = tc2 ^ z11;
 2703    S0 = tc3 ^ tc16;
 2704    S6 = ~(tc10 ^ tc18);
 2705    S4 = tc14 ^ S3;
 2706    S1 = ~(S3 ^ tc16);
 2707    tc26 = tc17 ^ tc20;
 2708    S2 = ~(tc26 ^ z17);
 2709    S5 = tc21 ^ tc17;
 2710
 2711    u[0] = S7;
 2712    u[1] = S6;
 2713    u[2] = S5;
 2714    u[3] = S4;
 2715    u[4] = S3;
 2716    u[5] = S2;
 2717    u[6] = S1;
 2718    u[7] = S0;
 2719}
 2720
 2721#define BS_MASK_BIT_SET(w, j, bmask) \
 2722    (((bs_word)0 - (((w) >> (j)) & (bs_word)1)) & (bmask))
 2723
 2724#define BS_TRANS_8(t, o, w, bmask, s)                   \
 2725    t[o + s + 0] |= BS_MASK_BIT_SET(w, s + 0, bmask);   \
 2726    t[o + s + 1] |= BS_MASK_BIT_SET(w, s + 1, bmask);   \
 2727    t[o + s + 2] |= BS_MASK_BIT_SET(w, s + 2, bmask);   \
 2728    t[o + s + 3] |= BS_MASK_BIT_SET(w, s + 3, bmask);   \
 2729    t[o + s + 4] |= BS_MASK_BIT_SET(w, s + 4, bmask);   \
 2730    t[o + s + 5] |= BS_MASK_BIT_SET(w, s + 5, bmask);   \
 2731    t[o + s + 6] |= BS_MASK_BIT_SET(w, s + 6, bmask);   \
 2732    t[o + s + 7] |= BS_MASK_BIT_SET(w, s + 7, bmask)
 2733
 2734static void bs_transpose(bs_word* t, bs_word* blocks)
 2735{
 2736    bs_word bmask = 1;
 2737    int i;
 2738
 2739    XMEMSET(t, 0, sizeof(bs_word) * AES_BLOCK_BITS);
 2740
 2741    for (i = 0; i < BS_WORD_SIZE; i++) {
 2742        int j;
 2743        int o = 0;
 2744        for (j = 0; j < BS_BLOCK_WORDS; j++) {
 2745        #ifdef LITTLE_ENDIAN_ORDER
 2746            bs_word w = blocks[i * BS_BLOCK_WORDS + j];
 2747        #else
 2748            bs_word w = bs_bswap(blocks[i * BS_BLOCK_WORDS + j]);
 2749        #endif
 2750    #ifdef WOLFSSL_AES_NO_UNROLL
 2751            int k;
 2752            for (k = 0; k < BS_WORD_SIZE; k++) {
 2753                t[o + k] |= BS_MASK_BIT_SET(w, k, bmask);
 2754            }
 2755    #else
 2756            BS_TRANS_8(t, o, w, bmask,  0);
 2757        #if BS_WORD_SIZE >= 16
 2758            BS_TRANS_8(t, o, w, bmask,  8);
 2759        #endif
 2760        #if BS_WORD_SIZE >= 32
 2761            BS_TRANS_8(t, o, w, bmask, 16);
 2762            BS_TRANS_8(t, o, w, bmask, 24);
 2763        #endif
 2764        #if BS_WORD_SIZE >= 64
 2765            BS_TRANS_8(t, o, w, bmask, 32);
 2766            BS_TRANS_8(t, o, w, bmask, 40);
 2767            BS_TRANS_8(t, o, w, bmask, 48);
 2768            BS_TRANS_8(t, o, w, bmask, 56);
 2769        #endif
 2770    #endif
 2771            o += BS_WORD_SIZE;
 2772        }
 2773        bmask <<= 1;
 2774    }
 2775}
 2776
 2777#define BS_INV_TRANS_8(t, o, w, bmask, s)                                   \
 2778    t[o + (s + 0) * BS_BLOCK_WORDS] |= BS_MASK_BIT_SET(w, s + 0, bmask);    \
 2779    t[o + (s + 1) * BS_BLOCK_WORDS] |= BS_MASK_BIT_SET(w, s + 1, bmask);    \
 2780    t[o + (s + 2) * BS_BLOCK_WORDS] |= BS_MASK_BIT_SET(w, s + 2, bmask);    \
 2781    t[o + (s + 3) * BS_BLOCK_WORDS] |= BS_MASK_BIT_SET(w, s + 3, bmask);    \
 2782    t[o + (s + 4) * BS_BLOCK_WORDS] |= BS_MASK_BIT_SET(w, s + 4, bmask);    \
 2783    t[o + (s + 5) * BS_BLOCK_WORDS] |= BS_MASK_BIT_SET(w, s + 5, bmask);    \
 2784    t[o + (s + 6) * BS_BLOCK_WORDS] |= BS_MASK_BIT_SET(w, s + 6, bmask);    \
 2785    t[o + (s + 7) * BS_BLOCK_WORDS] |= BS_MASK_BIT_SET(w, s + 7, bmask)
 2786
 2787static void bs_inv_transpose(bs_word* t, bs_word* blocks)
 2788{
 2789    int o;
 2790
 2791    XMEMSET(t, 0, sizeof(bs_word) * AES_BLOCK_BITS);
 2792
 2793    for (o = 0; o < BS_BLOCK_WORDS; o++) {
 2794        int i;
 2795        for (i = 0; i < BS_WORD_SIZE; i++) {
 2796        #ifdef LITTLE_ENDIAN_ORDER
 2797            bs_word bmask = (bs_word)1 << i;
 2798        #else
 2799            bs_word bmask = bs_bswap((bs_word)1 << i);
 2800        #endif
 2801            bs_word w = blocks[(o << BS_WORD_SHIFT) + i];
 2802    #ifdef WOLFSSL_AES_NO_UNROLL
 2803            int j;
 2804            for (j = 0; j < BS_WORD_SIZE; j++) {
 2805                t[j * BS_BLOCK_WORDS + o] |= BS_MASK_BIT_SET(w, j, bmask);
 2806            }
 2807    #else
 2808            BS_INV_TRANS_8(t, o, w, bmask, 0);
 2809        #if BS_WORD_SIZE >= 16
 2810            BS_INV_TRANS_8(t, o, w, bmask, 8);
 2811        #endif
 2812        #if BS_WORD_SIZE >= 32
 2813            BS_INV_TRANS_8(t, o, w, bmask, 16);
 2814            BS_INV_TRANS_8(t, o, w, bmask, 24);
 2815        #endif
 2816        #if BS_WORD_SIZE >= 64
 2817            BS_INV_TRANS_8(t, o, w, bmask, 32);
 2818            BS_INV_TRANS_8(t, o, w, bmask, 40);
 2819            BS_INV_TRANS_8(t, o, w, bmask, 48);
 2820            BS_INV_TRANS_8(t, o, w, bmask, 56);
 2821        #endif
 2822    #endif
 2823        }
 2824    }
 2825}
 2826
 2827#define BS_ROW_OFF_0    0
 2828#define BS_ROW_OFF_1    32
 2829#define BS_ROW_OFF_2    64
 2830#define BS_ROW_OFF_3    96
 2831
 2832#define BS_ROW_ADD      (AES_BLOCK_BITS / 16 + AES_BLOCK_BITS / 4)
 2833#define BS_IDX_MASK     0x7f
 2834
 2835#define BS_ASSIGN_8(d, od, s, os)   \
 2836    d[(od) + 0] = s[(os) + 0];      \
 2837    d[(od) + 1] = s[(os) + 1];      \
 2838    d[(od) + 2] = s[(os) + 2];      \
 2839    d[(od) + 3] = s[(os) + 3];      \
 2840    d[(od) + 4] = s[(os) + 4];      \
 2841    d[(od) + 5] = s[(os) + 5];      \
 2842    d[(od) + 6] = s[(os) + 6];      \
 2843    d[(od) + 7] = s[(os) + 7]
 2844
 2845static void bs_shift_rows(bs_word* t, bs_word* b)
 2846{
 2847    int i;
 2848
 2849    for (i = 0; i < 128; i += 32) {
 2850        BS_ASSIGN_8(t, i +  0, b, (  0 + i) & BS_IDX_MASK);
 2851        BS_ASSIGN_8(t, i +  8, b, ( 40 + i) & BS_IDX_MASK);
 2852        BS_ASSIGN_8(t, i + 16, b, ( 80 + i) & BS_IDX_MASK);
 2853        BS_ASSIGN_8(t, i + 24, b, (120 + i) & BS_IDX_MASK);
 2854    }
 2855}
 2856
 2857#define BS_SHIFT_OFF_0  0
 2858#define BS_SHIFT_OFF_1  8
 2859#define BS_SHIFT_OFF_2  16
 2860#define BS_SHIFT_OFF_3  24
 2861
 2862/* Shift rows and mix columns.
 2863 * See: See https://eprint.iacr.org/2009/129.pdf - Appendix A
 2864 */
 2865
 2866#define BS_SHIFT_MIX_8(t, o, br0, br1, br2, br3, of)                \
 2867        of      = br0[7] ^ br1[7];                                  \
 2868        t[o+0] =                   br1[0] ^ br2[0] ^ br3[0] ^ of;   \
 2869        t[o+1] = br0[0] ^ br1[0] ^ br1[1] ^ br2[1] ^ br3[1] ^ of;   \
 2870        t[o+2] = br0[1] ^ br1[1] ^ br1[2] ^ br2[2] ^ br3[2];        \
 2871        t[o+3] = br0[2] ^ br1[2] ^ br1[3] ^ br2[3] ^ br3[3] ^ of;   \
 2872        t[o+4] = br0[3] ^ br1[3] ^ br1[4] ^ br2[4] ^ br3[4] ^ of;   \
 2873        t[o+5] = br0[4] ^ br1[4] ^ br1[5] ^ br2[5] ^ br3[5];        \
 2874        t[o+6] = br0[5] ^ br1[5] ^ br1[6] ^ br2[6] ^ br3[6];        \
 2875        t[o+7] = br0[6] ^ br1[6] ^ br1[7] ^ br2[7] ^ br3[7]
 2876
 2877static void bs_shift_mix(bs_word* t, bs_word* b)
 2878{
 2879    int i;
 2880    word8 or0 = BS_ROW_OFF_0 + BS_SHIFT_OFF_0;
 2881    word8 or1 = BS_ROW_OFF_1 + BS_SHIFT_OFF_1;
 2882    word8 or2 = BS_ROW_OFF_2 + BS_SHIFT_OFF_2;
 2883    word8 or3 = BS_ROW_OFF_3 + BS_SHIFT_OFF_3;
 2884
 2885    for (i = 0; i < AES_BLOCK_BITS; i += AES_BLOCK_BITS / 4) {
 2886        bs_word* br0 = b + or0;
 2887        bs_word* br1 = b + or1;
 2888        bs_word* br2 = b + or2;
 2889        bs_word* br3 = b + or3;
 2890        bs_word of;
 2891
 2892        BS_SHIFT_MIX_8(t, i +  0, br0, br1, br2, br3, of);
 2893        BS_SHIFT_MIX_8(t, i +  8, br1, br2, br3, br0, of);
 2894        BS_SHIFT_MIX_8(t, i + 16, br2, br3, br0, br1, of);
 2895        BS_SHIFT_MIX_8(t, i + 24, br3, br0, br1, br2, of);
 2896
 2897        or0 = (or0 + AES_BLOCK_BITS / 4) & BS_IDX_MASK;
 2898        or1 = (or1 + AES_BLOCK_BITS / 4) & BS_IDX_MASK;
 2899        or2 = (or2 + AES_BLOCK_BITS / 4) & BS_IDX_MASK;
 2900        or3 = (or3 + AES_BLOCK_BITS / 4) & BS_IDX_MASK;
 2901    }
 2902}
 2903
 2904static void bs_add_round_key(bs_word* out, bs_word* b, bs_word* rk)
 2905{
 2906    xorbufout((byte*)out, (byte*)b, (byte*)rk, BS_BLOCK_SIZE);
 2907}
 2908
 2909static void bs_sub_bytes_blocks(bs_word* b)
 2910{
 2911    int i;
 2912
 2913    for (i = 0; i < AES_BLOCK_BITS; i += 8) {
 2914        bs_sub_bytes(b + i);
 2915    }
 2916}
 2917
 2918static const FLASH_QUALIFIER byte bs_rcon[] = {
 2919    0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1B, 0x36,
 2920    /* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
 2921};
 2922
 2923static void bs_ke_sub_bytes(unsigned char* out, unsigned char *in) {
 2924    bs_word block[AES_BLOCK_BITS];
 2925    bs_word trans[AES_BLOCK_BITS];
 2926
 2927    XMEMSET(block, 0, sizeof(block));
 2928    XMEMCPY(block, in, 4);
 2929
 2930    bs_transpose(trans, block);
 2931    bs_sub_bytes_blocks(trans);
 2932    bs_inv_transpose(block, trans);
 2933
 2934    XMEMCPY(out, block, 4);
 2935}
 2936
 2937static void bs_ke_transform(unsigned char* out, unsigned char *in, word8 i) {
 2938    /* Rotate the input 8 bits to the left */
 2939#ifdef LITTLE_ENDIAN_ORDER
 2940    *(word32*)out = rotrFixed(*(word32*)in, 8);
 2941#else
 2942    *(word32*)out = rotlFixed(*(word32*)in, 8);
 2943#endif
 2944    bs_ke_sub_bytes(out, out);
 2945    /* On just the first byte, add 2^i to the byte */
 2946    out[0] ^= bs_rcon[i];
 2947}
 2948
 2949static void bs_expand_key(unsigned char *in, word32 sz) {
 2950    unsigned char t[4];
 2951    word32 o;
 2952    word8 i = 0;
 2953
 2954    if (sz == 176) {
 2955        /* Total of 11 rounds - AES-128. */
 2956        for (o = 16; o < sz; o += 16) {
 2957            bs_ke_transform(t, in + o - 4, i);
 2958            i++;
 2959            *(word32*)(in + o +  0) = *(word32*)(in + o - 16) ^
 2960                                      *(word32*) t;
 2961            *(word32*)(in + o +  4) = *(word32*)(in + o - 12) ^
 2962                                      *(word32*)(in + o +  0);
 2963            *(word32*)(in + o +  8) = *(word32*)(in + o -  8) ^
 2964                                      *(word32*)(in + o +  4);
 2965            *(word32*)(in + o + 12) = *(word32*)(in + o -  4) ^
 2966                                      *(word32*)(in + o +  8);
 2967        }
 2968    }
 2969    else if (sz == 208) {
 2970        /* Total of 13 rounds - AES-192. */
 2971        for (o = 24; o < sz; o += 24) {
 2972            bs_ke_transform(t, in + o - 4, i);
 2973            i++;
 2974            *(word32*)(in + o +  0) = *(word32*)(in + o - 24) ^
 2975                                      *(word32*) t;
 2976            *(word32*)(in + o +  4) = *(word32*)(in + o - 20) ^
 2977                                      *(word32*)(in + o +  0);
 2978            *(word32*)(in + o +  8) = *(word32*)(in + o - 16) ^
 2979                                      *(word32*)(in + o +  4);
 2980            *(word32*)(in + o + 12) = *(word32*)(in + o - 12) ^
 2981                                      *(word32*)(in + o +  8);
 2982            *(word32*)(in + o + 16) = *(word32*)(in + o -  8) ^
 2983                                      *(word32*)(in + o + 12);
 2984            *(word32*)(in + o + 20) = *(word32*)(in + o -  4) ^
 2985                                      *(word32*)(in + o + 16);
 2986        }
 2987    }
 2988    else if (sz == 240) {
 2989        /* Total of 15 rounds - AES-256. */
 2990        for (o = 32; o < sz; o += 16) {
 2991            if ((o & 0x1f) == 0) {
 2992                bs_ke_transform(t, in + o - 4, i);
 2993                i++;
 2994            }
 2995            else {
 2996                bs_ke_sub_bytes(t, in + o - 4);
 2997            }
 2998            *(word32*)(in + o +  0) = *(word32*)(in + o - 32) ^
 2999                                      *(word32*) t;
 3000            *(word32*)(in + o +  4) = *(word32*)(in + o - 28) ^
 3001                                      *(word32*)(in + o +  0);
 3002            *(word32*)(in + o +  8) = *(word32*)(in + o - 24) ^
 3003                                      *(word32*)(in + o +  4);
 3004            *(word32*)(in + o + 12) = *(word32*)(in + o - 20) ^
 3005                                      *(word32*)(in + o +  8);
 3006        }
 3007    }
 3008}
 3009
 3010static void bs_set_key(bs_word* rk, const byte* key, word32 keyLen,
 3011    word32 rounds)
 3012{
 3013    int i;
 3014    byte bs_key[15 * WC_AES_BLOCK_SIZE];
 3015    int ksSz = (rounds + 1) * WC_AES_BLOCK_SIZE;
 3016    bs_word block[AES_BLOCK_BITS];
 3017
 3018    /* Fist round. */
 3019    XMEMCPY(bs_key, key, keyLen);
 3020    bs_expand_key(bs_key, ksSz);
 3021
 3022    for (i = 0; i < ksSz; i += WC_AES_BLOCK_SIZE) {
 3023        int k;
 3024
 3025        XMEMCPY(block, bs_key + i, WC_AES_BLOCK_SIZE);
 3026        for (k = BS_BLOCK_WORDS; k < AES_BLOCK_BITS; k += BS_BLOCK_WORDS) {
 3027            int l;
 3028            for (l = 0; l < BS_BLOCK_WORDS; l++) {
 3029                block[k + l] = block[l];
 3030            }
 3031        }
 3032        bs_transpose(rk, block);
 3033        rk += AES_BLOCK_BITS;
 3034    }
 3035}
 3036
 3037static void bs_encrypt(bs_word* state, bs_word* rk, word32 r)
 3038{
 3039    word32 i;
 3040    bs_word trans[AES_BLOCK_BITS];
 3041
 3042    bs_transpose(trans, state);
 3043
 3044    bs_add_round_key(trans, trans, rk);
 3045    for (i = 1; i < r; i++) {
 3046        bs_sub_bytes_blocks(trans);
 3047        bs_shift_mix(state, trans);
 3048        rk += AES_BLOCK_BITS;
 3049        bs_add_round_key(trans, state, rk);
 3050    }
 3051    bs_sub_bytes_blocks(trans);
 3052    bs_shift_rows(state, trans);
 3053    rk += AES_BLOCK_BITS;
 3054    bs_add_round_key(trans, state, rk);
 3055    bs_inv_transpose(state, trans);
 3056}
 3057
 3058#ifndef HAVE_CUDA
 3059/* Encrypt a block using AES.
 3060 *
 3061 * @param [in]  aes       AES object.
 3062 * @param [in]  inBlock   Block to encrypt.
 3063 * @param [out] outBlock  Encrypted block.
 3064 * @param [in]  r         Rounds divided by 2.
 3065 */
 3066static void AesEncrypt_C(Aes* aes, const byte* inBlock, byte* outBlock,
 3067        word32 r)
 3068{
 3069    bs_word state[AES_BLOCK_BITS];
 3070
 3071    (void)r;
 3072
 3073    XMEMCPY(state, inBlock, WC_AES_BLOCK_SIZE);
 3074    XMEMSET(((byte*)state) + WC_AES_BLOCK_SIZE, 0, sizeof(state) - WC_AES_BLOCK_SIZE);
 3075
 3076    bs_encrypt(state, aes->bs_key, aes->rounds);
 3077
 3078    XMEMCPY(outBlock, state, WC_AES_BLOCK_SIZE);
 3079}
 3080
 3081#if defined(HAVE_AES_ECB) && !(defined(WOLFSSL_IMX6_CAAM) && \
 3082    !defined(NO_IMX6_CAAM_AES) && !defined(WOLFSSL_QNX_CAAM))
 3083/* Encrypt a number of blocks using AES.
 3084 *
 3085 * @param [in]  aes  AES object.
 3086 * @param [in]  in   Block to encrypt.
 3087 * @param [out] out  Encrypted block.
 3088 * @param [in]  sz   Number of blocks to encrypt.
 3089 */
 3090static void AesEncryptBlocks_C(Aes* aes, const byte* in, byte* out, word32 sz)
 3091{
 3092    bs_word state[AES_BLOCK_BITS];
 3093
 3094    while (sz >= BS_BLOCK_SIZE) {
 3095        XMEMCPY(state, in, BS_BLOCK_SIZE);
 3096        bs_encrypt(state, aes->bs_key, aes->rounds);
 3097        XMEMCPY(out, state, BS_BLOCK_SIZE);
 3098        sz  -= BS_BLOCK_SIZE;
 3099        in  += BS_BLOCK_SIZE;
 3100        out += BS_BLOCK_SIZE;
 3101    }
 3102    if (sz > 0) {
 3103        XMEMCPY(state, in, sz);
 3104        XMEMSET(((byte*)state) + sz, 0, sizeof(state) - sz);
 3105        bs_encrypt(state, aes->bs_key, aes->rounds);
 3106        XMEMCPY(out, state, sz);
 3107    }
 3108}
 3109#endif
 3110#else
 3111extern void AesEncrypt_C(Aes* aes, const byte* inBlock, byte* outBlock,
 3112        word32 r);
 3113extern void AesEncryptBlocks_C(Aes* aes, const byte* in, byte* out, word32 sz);
 3114#endif /* HAVE_CUDA */
 3115
 3116#endif /* !WC_AES_BITSLICED */
 3117
 3118#ifdef WC_AES_HAVE_PREFETCH_ARG
 3119#define wc_AesEncrypt(aes, inBlock, outBlock) \
 3120    AesEncrypt_preFetchOpt(aes, inBlock, outBlock, &always_prefetch)
 3121WC_ALL_ARGS_NOT_NULL static WARN_UNUSED_RESULT int AesEncrypt_preFetchOpt(
 3122    Aes* aes, const byte* inBlock, byte* outBlock, int *prefetch_ptr)
 3123#else
 3124#define AesEncrypt_preFetchOpt(aes, inBlock, outBlock, prefetch_ptr) \
 3125    wc_AesEncrypt(aes, inBlock, outBlock)
 3126WC_ALL_ARGS_NOT_NULL static WARN_UNUSED_RESULT int wc_AesEncrypt(
 3127    Aes* aes, const byte* inBlock, byte* outBlock)
 3128#endif
 3129{
 3130#if defined(MAX3266X_AES)
 3131    word32 keySize;
 3132#endif
 3133#if defined(MAX3266X_CB)
 3134    int ret_cb;
 3135#endif
 3136    word32 r;
 3137
 3138#ifdef WC_DEBUG_CIPHER_LIFECYCLE
 3139    {
 3140        int ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
 3141        if (ret < 0)
 3142            return ret;
 3143    }
 3144#endif
 3145
 3146    r = aes->rounds >> 1;
 3147
 3148    if (r > 7 || r == 0) {
 3149        WOLFSSL_ERROR_VERBOSE(KEYUSAGE_E);
 3150        return KEYUSAGE_E;
 3151    }
 3152
 3153#ifdef WOLFSSL_AESNI
 3154    if (aes->use_aesni) {
 3155        ASSERT_SAVED_VECTOR_REGISTERS();
 3156
 3157        #ifdef DEBUG_AESNI
 3158            printf("about to aes encrypt\n");
 3159            printf("in  = %p\n", inBlock);
 3160            printf("out = %p\n", outBlock);
 3161            printf("aes->key = %p\n", aes->key);
 3162            printf("aes->rounds = %d\n", aes->rounds);
 3163            printf("sz = %d\n", WC_AES_BLOCK_SIZE);
 3164        #endif
 3165
 3166        /* check alignment, decrypt doesn't need alignment */
 3167        if ((wc_ptr_t)inBlock % AESNI_ALIGN) {
 3168        #ifndef NO_WOLFSSL_ALLOC_ALIGN
 3169            byte* tmp = (byte*)XMALLOC(WC_AES_BLOCK_SIZE + AESNI_ALIGN, aes->heap,
 3170                                                      DYNAMIC_TYPE_TMP_BUFFER);
 3171            byte* tmp_align;
 3172            if (tmp == NULL)
 3173                return MEMORY_E;
 3174
 3175            tmp_align = tmp + (AESNI_ALIGN - ((wc_ptr_t)tmp % AESNI_ALIGN));
 3176
 3177            XMEMCPY(tmp_align, inBlock, WC_AES_BLOCK_SIZE);
 3178            AES_ECB_encrypt_AESNI(tmp_align, tmp_align, WC_AES_BLOCK_SIZE,
 3179                    (byte*)aes->key, (int)aes->rounds);
 3180            XMEMCPY(outBlock, tmp_align, WC_AES_BLOCK_SIZE);
 3181            XFREE(tmp, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
 3182            return 0;
 3183        #else
 3184            WOLFSSL_MSG("AES-ECB encrypt with bad alignment");
 3185            WOLFSSL_ERROR_VERBOSE(BAD_ALIGN_E);
 3186            return BAD_ALIGN_E;
 3187        #endif
 3188        }
 3189
 3190        AES_ECB_encrypt_AESNI(inBlock, outBlock, WC_AES_BLOCK_SIZE, (byte*)aes->key,
 3191                        (int)aes->rounds);
 3192
 3193        return 0;
 3194    }
 3195    else {
 3196        #ifdef DEBUG_AESNI
 3197            printf("Skipping AES-NI\n");
 3198        #endif
 3199    }
 3200#elif defined(WOLFSSL_ARMASM)
 3201#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
 3202#if !defined(__aarch64__)
 3203    AES_encrypt_AARCH32(inBlock, outBlock, (byte*)aes->key, (int)aes->rounds);
 3204#else
 3205    if (aes->use_aes_hw_crypto) {
 3206        AES_encrypt_AARCH64(inBlock, outBlock, (byte*)aes->key,
 3207            (int)aes->rounds);
 3208    }
 3209    else
 3210#endif /* !__aarch64__ */
 3211#endif /* !WOLFSSL_ARMASM_NO_HW_CRYPTO */
 3212#if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON) && \
 3213    defined(WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP)
 3214    {
 3215        AES_ECB_encrypt_NEON(inBlock, outBlock, WC_AES_BLOCK_SIZE,
 3216            (const unsigned char*)aes->key, aes->rounds);
 3217    }
 3218#elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
 3219    {
 3220        AES_ECB_encrypt(inBlock, outBlock, WC_AES_BLOCK_SIZE,
 3221            (const unsigned char*)aes->key, aes->rounds);
 3222    }
 3223#endif
 3224    return 0;
 3225#endif /* WOLFSSL_AESNI */
 3226#if defined(WOLFSSL_SCE) && !defined(WOLFSSL_SCE_NO_AES)
 3227    AES_ECB_encrypt(aes, inBlock, outBlock, WC_AES_BLOCK_SIZE);
 3228    return 0;
 3229#endif
 3230
 3231#if defined(WOLFSSL_IMXRT_DCP)
 3232    if (aes->keylen == 16) {
 3233        DCPAesEcbEncrypt(aes, outBlock, inBlock, WC_AES_BLOCK_SIZE);
 3234        return 0;
 3235    }
 3236#endif
 3237
 3238#if defined(WOLFSSL_SE050) && defined(WOLFSSL_SE050_CRYPT)
 3239    if (aes->useSWCrypt == 0) {
 3240        return se050_aes_crypt(aes, inBlock, outBlock, WC_AES_BLOCK_SIZE,
 3241                               AES_ENCRYPTION, kAlgorithm_SSS_AES_ECB);
 3242    }
 3243#endif
 3244
 3245#if defined(WOLFSSL_ESPIDF) && defined(NEED_AES_HW_FALLBACK)
 3246    ESP_LOGV(TAG, "wc_AesEncrypt fallback check");
 3247    if (wc_esp32AesSupportedKeyLen(aes)) {
 3248        return wc_esp32AesEncrypt(aes, inBlock, outBlock);
 3249    }
 3250    else {
 3251        /* For example, the ESP32-S3 does not support HW for len = 24,
 3252         * so fall back to SW */
 3253    #ifdef DEBUG_WOLFSSL
 3254        ESP_LOGW(TAG, "wc_AesEncrypt HW Falling back, unsupported keylen = %d",
 3255                      aes->keylen);
 3256    #endif
 3257    }
 3258#endif
 3259
 3260#if defined(MAX3266X_AES)
 3261    if (wc_AesGetKeySize(aes, &keySize) == 0) {
 3262        return wc_MXC_TPU_AesEncrypt(inBlock, (byte*)aes->reg, (byte*)aes->key,
 3263                                    MXC_TPU_MODE_ECB, WC_AES_BLOCK_SIZE,
 3264                                    outBlock, (unsigned int)keySize);
 3265    }
 3266#endif
 3267#if defined(MAX3266X_CB) && defined(HAVE_AES_ECB) /* Can do a basic ECB block */
 3268    #ifndef WOLF_CRYPTO_CB_FIND
 3269    if (aes->devId != INVALID_DEVID)
 3270    #endif
 3271    {
 3272        ret_cb = wc_CryptoCb_AesEcbEncrypt(aes, outBlock, inBlock,
 3273                                            WC_AES_BLOCK_SIZE);
 3274        if (ret_cb != WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE))
 3275            return ret_cb;
 3276        /* fall-through when unavailable */
 3277    }
 3278#endif
 3279
 3280#ifdef WC_AES_HAVE_PREFETCH_ARG
 3281    AesEncrypt_C(aes, inBlock, outBlock, r, prefetch_ptr);
 3282#else
 3283    AesEncrypt_C(aes, inBlock, outBlock, r);
 3284#endif
 3285
 3286    return 0;
 3287} /* wc_AesEncrypt */
 3288#endif
 3289#endif /* HAVE_AES_CBC || WOLFSSL_AES_DIRECT || HAVE_AESGCM */
 3290
 3291#if defined(HAVE_AES_DECRYPT)
 3292#if ((defined(HAVE_AES_CBC) && !defined(WOLFSSL_DEVCRYPTO_CBC)) || \
 3293     defined(HAVE_AES_ECB) || defined(WOLFSSL_AES_DIRECT)) && \
 3294    (defined(__aarch64__) || !defined(WOLFSSL_ARMASM))
 3295
 3296#ifndef WC_AES_BITSLICED
 3297#ifndef WC_NO_CACHE_RESISTANT
 3298#ifndef WOLFSSL_AES_SMALL_TABLES
 3299/* load 4 Td Tables into cache by cache line stride */
 3300static WARN_UNUSED_RESULT WC_INLINE word32 PreFetchTd(void)
 3301{
 3302    volatile word32 x = 0;
 3303    int i;
 3304    int j;
 3305
 3306    for (i = 0; i < 4; i++) {
 3307        /* 256 elements, each one is 4 bytes */
 3308        for (j = 0; j < 256; j += WC_CACHE_LINE_SZ / 4) {
 3309            x &= Td[i][j];
 3310        }
 3311    }
 3312
 3313    return x;
 3314}
 3315#endif /* !WOLFSSL_AES_SMALL_TABLES */
 3316
 3317/* load Td Table4 into cache by cache line stride */
 3318static WARN_UNUSED_RESULT WC_INLINE word32 PreFetchTd4(void)
 3319{
 3320#ifndef WOLFSSL_AES_TOUCH_LINES
 3321    volatile word32 x = 0;
 3322    int i;
 3323
 3324    for (i = 0; i < 256; i += WC_CACHE_LINE_SZ) {
 3325        x &= (word32)Td4[i];
 3326    }
 3327
 3328    return x;
 3329#else
 3330    return 0;
 3331#endif
 3332}
 3333#endif /* !WC_NO_CACHE_RESISTANT */
 3334
 3335/* Decrypt a block using AES.
 3336 *
 3337 * @param [in]  aes       AES object.
 3338 * @param [in]  inBlock   Block to encrypt.
 3339 * @param [out] outBlock  Encrypted block.
 3340 * @param [in]  r         Rounds divided by 2.
 3341 */
 3342#ifndef WC_AES_HAVE_PREFETCH_ARG
 3343    #define WC_AES_HAVE_PREFETCH_ARG
 3344    static int always_prefetch = 0;
 3345    WC_MAYBE_UNUSED static int never_prefetch = 1;
 3346#endif
 3347WC_ARGS_NOT_NULL((1, 2, 3, 5))
 3348static void AesDecrypt_C(Aes* aes, const byte* inBlock, byte* outBlock,
 3349    word32 r, int *prefetch_ptr)
 3350{
 3351    word32 s0 = 0, s1 = 0, s2 = 0, s3 = 0;
 3352    word32 t0 = 0, t1 = 0, t2 = 0, t3 = 0;
 3353    const word32* rk;
 3354
 3355#ifdef WC_C_DYNAMIC_FALLBACK
 3356    rk = aes->key_C_fallback;
 3357#else
 3358    rk = aes->key;
 3359#endif
 3360
 3361    /*
 3362     * map byte array block to cipher state
 3363     * and add initial round key:
 3364     */
 3365    XMEMCPY(&s0, inBlock,                  sizeof(s0));
 3366    XMEMCPY(&s1, inBlock + sizeof(s0),     sizeof(s1));
 3367    XMEMCPY(&s2, inBlock + 2 * sizeof(s0), sizeof(s2));
 3368    XMEMCPY(&s3, inBlock + 3 * sizeof(s0), sizeof(s3));
 3369
 3370#ifdef LITTLE_ENDIAN_ORDER
 3371    s0 = ByteReverseWord32(s0);
 3372    s1 = ByteReverseWord32(s1);
 3373    s2 = ByteReverseWord32(s2);
 3374    s3 = ByteReverseWord32(s3);
 3375#endif
 3376
 3377    s0 ^= rk[0];
 3378    s1 ^= rk[1];
 3379    s2 ^= rk[2];
 3380    s3 ^= rk[3];
 3381
 3382#ifndef WOLFSSL_AES_SMALL_TABLES
 3383
 3384#ifndef WC_NO_CACHE_RESISTANT
 3385    if (*prefetch_ptr == 0) {
 3386        s0 |= PreFetchTd();
 3387        /* don't set the prefetched flag here -- PreFetchTd4() is called
 3388         * below.
 3389         */
 3390    }
 3391#else
 3392    (void)prefetch_ptr;
 3393#endif
 3394
 3395#ifndef WOLFSSL_AES_TOUCH_LINES
 3396/* Unroll the loop. */
 3397#define DEC_ROUND_T_S(o)                                            \
 3398    t0 = GetTable(Td[0], GETBYTE(s0, 3)) ^ GetTable(Td[1], GETBYTE(s3, 2)) ^            \
 3399         GetTable(Td[2], GETBYTE(s2, 1)) ^ GetTable(Td[3], GETBYTE(s1, 0)) ^ rk[(o)+4]; \
 3400    t1 = GetTable(Td[0], GETBYTE(s1, 3)) ^ GetTable(Td[1], GETBYTE(s0, 2)) ^            \
 3401         GetTable(Td[2], GETBYTE(s3, 1)) ^ GetTable(Td[3], GETBYTE(s2, 0)) ^ rk[(o)+5]; \
 3402    t2 = GetTable(Td[0], GETBYTE(s2, 3)) ^ GetTable(Td[1], GETBYTE(s1, 2)) ^            \
 3403         GetTable(Td[2], GETBYTE(s0, 1)) ^ GetTable(Td[3], GETBYTE(s3, 0)) ^ rk[(o)+6]; \
 3404    t3 = GetTable(Td[0], GETBYTE(s3, 3)) ^ GetTable(Td[1], GETBYTE(s2, 2)) ^            \
 3405         GetTable(Td[2], GETBYTE(s1, 1)) ^ GetTable(Td[3], GETBYTE(s0, 0)) ^ rk[(o)+7]
 3406#define DEC_ROUND_S_T(o)                                            \
 3407    s0 = GetTable(Td[0], GETBYTE(t0, 3)) ^ GetTable(Td[1], GETBYTE(t3, 2)) ^            \
 3408         GetTable(Td[2], GETBYTE(t2, 1)) ^ GetTable(Td[3], GETBYTE(t1, 0)) ^ rk[(o)+0]; \
 3409    s1 = GetTable(Td[0], GETBYTE(t1, 3)) ^ GetTable(Td[1], GETBYTE(t0, 2)) ^            \
 3410         GetTable(Td[2], GETBYTE(t3, 1)) ^ GetTable(Td[3], GETBYTE(t2, 0)) ^ rk[(o)+1]; \
 3411    s2 = GetTable(Td[0], GETBYTE(t2, 3)) ^ GetTable(Td[1], GETBYTE(t1, 2)) ^            \
 3412         GetTable(Td[2], GETBYTE(t0, 1)) ^ GetTable(Td[3], GETBYTE(t3, 0)) ^ rk[(o)+2]; \
 3413    s3 = GetTable(Td[0], GETBYTE(t3, 3)) ^ GetTable(Td[1], GETBYTE(t2, 2)) ^            \
 3414         GetTable(Td[2], GETBYTE(t1, 1)) ^ GetTable(Td[3], GETBYTE(t0, 0)) ^ rk[(o)+3]
 3415#else
 3416#define DEC_ROUND_T_S(o)                                                       \
 3417    GetTable_Multi(Td[0], &t0, GETBYTE(s0, 3), &t1, GETBYTE(s1, 3),            \
 3418                          &t2, GETBYTE(s2, 3), &t3, GETBYTE(s3, 3));           \
 3419    XorTable_Multi(Td[1], &t0, GETBYTE(s3, 2), &t1, GETBYTE(s0, 2),            \
 3420                          &t2, GETBYTE(s1, 2), &t3, GETBYTE(s2, 2));           \
 3421    XorTable_Multi(Td[2], &t0, GETBYTE(s2, 1), &t1, GETBYTE(s3, 1),            \
 3422                          &t2, GETBYTE(s0, 1), &t3, GETBYTE(s1, 1));           \
 3423    XorTable_Multi(Td[3], &t0, GETBYTE(s1, 0), &t1, GETBYTE(s2, 0),            \
 3424                          &t2, GETBYTE(s3, 0), &t3, GETBYTE(s0, 0));           \
 3425    t0 ^= rk[(o)+4]; t1 ^= rk[(o)+5]; t2 ^= rk[(o)+6]; t3 ^= rk[(o)+7];
 3426
 3427#define DEC_ROUND_S_T(o)                                                       \
 3428    GetTable_Multi(Td[0], &s0, GETBYTE(t0, 3), &s1, GETBYTE(t1, 3),            \
 3429                          &s2, GETBYTE(t2, 3), &s3, GETBYTE(t3, 3));           \
 3430    XorTable_Multi(Td[1], &s0, GETBYTE(t3, 2), &s1, GETBYTE(t0, 2),            \
 3431                          &s2, GETBYTE(t1, 2), &s3, GETBYTE(t2, 2));           \
 3432    XorTable_Multi(Td[2], &s0, GETBYTE(t2, 1), &s1, GETBYTE(t3, 1),            \
 3433                          &s2, GETBYTE(t0, 1), &s3, GETBYTE(t1, 1));           \
 3434    XorTable_Multi(Td[3], &s0, GETBYTE(t1, 0), &s1, GETBYTE(t2, 0),            \
 3435                          &s2, GETBYTE(t3, 0), &s3, GETBYTE(t0, 0));           \
 3436    s0 ^= rk[(o)+0]; s1 ^= rk[(o)+1]; s2 ^= rk[(o)+2]; s3 ^= rk[(o)+3];
 3437#endif
 3438
 3439#ifndef WOLFSSL_AES_NO_UNROLL
 3440                       DEC_ROUND_T_S( 0);
 3441    DEC_ROUND_S_T( 8); DEC_ROUND_T_S( 8);
 3442    DEC_ROUND_S_T(16); DEC_ROUND_T_S(16);
 3443    DEC_ROUND_S_T(24); DEC_ROUND_T_S(24);
 3444    DEC_ROUND_S_T(32); DEC_ROUND_T_S(32);
 3445    if (r > 5) {
 3446        DEC_ROUND_S_T(40); DEC_ROUND_T_S(40);
 3447        if (r > 6) {
 3448            DEC_ROUND_S_T(48); DEC_ROUND_T_S(48);
 3449        }
 3450    }
 3451    rk += r * 8;
 3452#else
 3453
 3454    /*
 3455     * Nr - 1 full rounds:
 3456     */
 3457
 3458    for (;;) {
 3459        DEC_ROUND_T_S(0);
 3460
 3461        rk += 8;
 3462        if (--r == 0) {
 3463            break;
 3464        }
 3465
 3466        DEC_ROUND_S_T(0);
 3467    }
 3468#endif
 3469    /*
 3470     * apply last round and
 3471     * map cipher state to byte array block:
 3472     */
 3473
 3474#ifndef WC_NO_CACHE_RESISTANT
 3475    if (*prefetch_ptr == 0) {
 3476        t0 |= PreFetchTd4();
 3477        if (prefetch_ptr != &always_prefetch)
 3478            *prefetch_ptr = 1;
 3479    }
 3480#else
 3481    (void)prefetch_ptr;
 3482#endif
 3483
 3484    s0 = GetTable8_4(Td4, GETBYTE(t0, 3), GETBYTE(t3, 2),
 3485                          GETBYTE(t2, 1), GETBYTE(t1, 0)) ^ rk[0];
 3486    s1 = GetTable8_4(Td4, GETBYTE(t1, 3), GETBYTE(t0, 2),
 3487                          GETBYTE(t3, 1), GETBYTE(t2, 0)) ^ rk[1];
 3488    s2 = GetTable8_4(Td4, GETBYTE(t2, 3), GETBYTE(t1, 2),
 3489                          GETBYTE(t0, 1), GETBYTE(t3, 0)) ^ rk[2];
 3490    s3 = GetTable8_4(Td4, GETBYTE(t3, 3), GETBYTE(t2, 2),
 3491                          GETBYTE(t1, 1), GETBYTE(t0, 0)) ^ rk[3];
 3492
 3493#else /* WOLFSSL_AES_SMALL_TABLES */
 3494
 3495#ifndef WC_NO_CACHE_RESISTANT
 3496    if (*prefetch_ptr == 0) {
 3497        s0 |= PreFetchTd4();
 3498        if (prefetch_ptr != &always_prefetch)
 3499            *prefetch_ptr = 1;
 3500    }
 3501#else
 3502    (void)prefetch_ptr;
 3503#endif
 3504
 3505    r *= 2;
 3506    for (rk += 4; r > 1; r--, rk += 4) {
 3507        t0 =
 3508            ((word32)GetTable8(Td4, GETBYTE(s0, 3)) << 24) ^
 3509            ((word32)GetTable8(Td4, GETBYTE(s3, 2)) << 16) ^
 3510            ((word32)GetTable8(Td4, GETBYTE(s2, 1)) <<  8) ^
 3511            ((word32)GetTable8(Td4, GETBYTE(s1, 0))) ^
 3512            rk[0];
 3513        t1 =
 3514            ((word32)GetTable8(Td4, GETBYTE(s1, 3)) << 24) ^
 3515            ((word32)GetTable8(Td4, GETBYTE(s0, 2)) << 16) ^
 3516            ((word32)GetTable8(Td4, GETBYTE(s3, 1)) <<  8) ^
 3517            ((word32)GetTable8(Td4, GETBYTE(s2, 0))) ^
 3518            rk[1];
 3519        t2 =
 3520            ((word32)GetTable8(Td4, GETBYTE(s2, 3)) << 24) ^
 3521            ((word32)GetTable8(Td4, GETBYTE(s1, 2)) << 16) ^
 3522            ((word32)GetTable8(Td4, GETBYTE(s0, 1)) <<  8) ^
 3523            ((word32)GetTable8(Td4, GETBYTE(s3, 0))) ^
 3524            rk[2];
 3525        t3 =
 3526            ((word32)GetTable8(Td4, GETBYTE(s3, 3)) << 24) ^
 3527            ((word32)GetTable8(Td4, GETBYTE(s2, 2)) << 16) ^
 3528            ((word32)GetTable8(Td4, GETBYTE(s1, 1)) <<  8) ^
 3529            ((word32)GetTable8(Td4, GETBYTE(s0, 0))) ^
 3530            rk[3];
 3531
 3532        s0 =
 3533            (inv_col_mul(t0, 0, 2, 1, 3) << 24) ^
 3534            (inv_col_mul(t0, 3, 1, 0, 2) << 16) ^
 3535            (inv_col_mul(t0, 2, 0, 3, 1) <<  8) ^
 3536            (inv_col_mul(t0, 1, 3, 2, 0)      );
 3537        s1 =
 3538            (inv_col_mul(t1, 0, 2, 1, 3) << 24) ^
 3539            (inv_col_mul(t1, 3, 1, 0, 2) << 16) ^
 3540            (inv_col_mul(t1, 2, 0, 3, 1) <<  8) ^
 3541            (inv_col_mul(t1, 1, 3, 2, 0)      );
 3542        s2 =
 3543            (inv_col_mul(t2, 0, 2, 1, 3) << 24) ^
 3544            (inv_col_mul(t2, 3, 1, 0, 2) << 16) ^
 3545            (inv_col_mul(t2, 2, 0, 3, 1) <<  8) ^
 3546            (inv_col_mul(t2, 1, 3, 2, 0)      );
 3547        s3 =
 3548            (inv_col_mul(t3, 0, 2, 1, 3) << 24) ^
 3549            (inv_col_mul(t3, 3, 1, 0, 2) << 16) ^
 3550            (inv_col_mul(t3, 2, 0, 3, 1) <<  8) ^
 3551            (inv_col_mul(t3, 1, 3, 2, 0)      );
 3552    }
 3553
 3554    t0 =
 3555        ((word32)GetTable8(Td4, GETBYTE(s0, 3)) << 24) ^
 3556        ((word32)GetTable8(Td4, GETBYTE(s3, 2)) << 16) ^
 3557        ((word32)GetTable8(Td4, GETBYTE(s2, 1)) <<  8) ^
 3558        ((word32)GetTable8(Td4, GETBYTE(s1, 0)));
 3559    t1 =
 3560        ((word32)GetTable8(Td4, GETBYTE(s1, 3)) << 24) ^
 3561        ((word32)GetTable8(Td4, GETBYTE(s0, 2)) << 16) ^
 3562        ((word32)GetTable8(Td4, GETBYTE(s3, 1)) <<  8) ^
 3563        ((word32)GetTable8(Td4, GETBYTE(s2, 0)));
 3564    t2 =
 3565        ((word32)GetTable8(Td4, GETBYTE(s2, 3)) << 24) ^
 3566        ((word32)GetTable8(Td4, GETBYTE(s1, 2)) << 16) ^
 3567        ((word32)GetTable8(Td4, GETBYTE(s0, 1)) <<  8) ^
 3568        ((word32)GetTable8(Td4, GETBYTE(s3, 0)));
 3569    t3 =
 3570        ((word32)GetTable8(Td4, GETBYTE(s3, 3)) << 24) ^
 3571        ((word32)GetTable8(Td4, GETBYTE(s2, 2)) << 16) ^
 3572        ((word32)GetTable8(Td4, GETBYTE(s1, 1)) <<  8) ^
 3573        ((word32)GetTable8(Td4, GETBYTE(s0, 0)));
 3574    s0 = t0 ^ rk[0];
 3575    s1 = t1 ^ rk[1];
 3576    s2 = t2 ^ rk[2];
 3577    s3 = t3 ^ rk[3];
 3578
 3579#endif /* WOLFSSL_AES_SMALL_TABLES */
 3580
 3581    /* write out */
 3582#ifdef LITTLE_ENDIAN_ORDER
 3583    s0 = ByteReverseWord32(s0);
 3584    s1 = ByteReverseWord32(s1);
 3585    s2 = ByteReverseWord32(s2);
 3586    s3 = ByteReverseWord32(s3);
 3587#endif
 3588
 3589    XMEMCPY(outBlock,                  &s0, sizeof(s0));
 3590    XMEMCPY(outBlock + sizeof(s0),     &s1, sizeof(s1));
 3591    XMEMCPY(outBlock + 2 * sizeof(s0), &s2, sizeof(s2));
 3592    XMEMCPY(outBlock + 3 * sizeof(s0), &s3, sizeof(s3));
 3593
 3594}
 3595
 3596#if defined(HAVE_AES_ECB) && !(defined(WOLFSSL_IMX6_CAAM) && \
 3597    !defined(NO_IMX6_CAAM_AES) && !defined(WOLFSSL_QNX_CAAM)) && \
 3598    !defined(MAX3266X_AES)
 3599#if defined(__aarch64__) || !defined(WOLFSSL_ARMASM)
 3600/* Decrypt a number of blocks using AES.
 3601 *
 3602 * @param [in]  aes  AES object.
 3603 * @param [in]  in   Block to encrypt.
 3604 * @param [out] out  Encrypted block.
 3605 * @param [in]  sz   Number of blocks to encrypt.
 3606 */
 3607static void AesDecryptBlocks_C(Aes* aes, const byte* in, byte* out, word32 sz)
 3608{
 3609    word32 i;
 3610    int did_prefetches = 0;
 3611
 3612    for (i = 0; i < sz; i += WC_AES_BLOCK_SIZE) {
 3613        AesDecrypt_C(aes, in, out, aes->rounds >> 1, &did_prefetches);
 3614        in += WC_AES_BLOCK_SIZE;
 3615        out += WC_AES_BLOCK_SIZE;
 3616    }
 3617}
 3618#endif
 3619#endif
 3620
 3621#else /* WC_AES_BITSLICED */
 3622
 3623/* http://cs-www.cs.yale.edu/homes/peralta/CircuitStuff/Sinv.txt */
 3624static void bs_inv_sub_bytes(bs_word u[8])
 3625{
 3626    bs_word U0, U1, U2, U3, U4, U5, U6, U7;
 3627    bs_word Y0, Y1, Y2, Y3, Y4, Y5, Y6, Y7;
 3628    bs_word RTL0, RTL1, RTL2;
 3629    bs_word sa0, sa1;
 3630    bs_word sb0, sb1;
 3631    bs_word ab0, ab1, ab2, ab3;
 3632    bs_word ab20, ab21, ab22, ab23;
 3633    bs_word al, ah, aa, bl, bh, bb;
 3634    bs_word abcd1, abcd2, abcd3, abcd4, abcd5, abcd6;
 3635    bs_word ph11, ph12, ph13, ph01, ph02, ph03;
 3636    bs_word pl01, pl02, pl03, pl11, pl12, pl13;
 3637    bs_word r1, r2, r3, r4, r5, r6, r7, r8, r9;
 3638    bs_word rr1, rr2;
 3639    bs_word r10, r11;
 3640    bs_word cp1, cp2, cp3, cp4;
 3641    bs_word vr1, vr2, vr3;
 3642    bs_word pr1, pr2, pr3;
 3643    bs_word wr1, wr2, wr3;
 3644    bs_word qr1, qr2, qr3;
 3645    bs_word tinv1, tinv2, tinv3, tinv4, tinv5, tinv6, tinv7, tinv8, tinv9;
 3646    bs_word tinv10, tinv11, tinv12, tinv13;
 3647    bs_word t01, t02;
 3648    bs_word d0, d1, d2, d3;
 3649    bs_word dl, dd, dh;
 3650    bs_word sd0, sd1;
 3651    bs_word p0, p1, p2, p3, p4, p6, p7;
 3652    bs_word X11, X13, X14, X16, X18, X19;
 3653    bs_word S0, S1, S2, S3, S4, S5, S6, S7;
 3654
 3655    U0 = u[7];
 3656    U1 = u[6];
 3657    U2 = u[5];
 3658    U3 = u[4];
 3659    U4 = u[3];
 3660    U5 = u[2];
 3661    U6 = u[1];
 3662    U7 = u[0];
 3663
 3664    Y0 = U0 ^ U3;
 3665    Y2 = ~(U1 ^ U3);
 3666    Y4 = U0 ^ Y2;
 3667    RTL0 = U6 ^ U7;
 3668    Y1 = Y2 ^ RTL0;
 3669    Y7 = ~(U2 ^ Y1);
 3670    RTL1 = U3 ^ U4;
 3671    Y6 = ~(U7 ^ RTL1);
 3672    Y3 = Y1 ^ RTL1;
 3673    RTL2 = ~(U0 ^ U2);
 3674    Y5 = U5 ^ RTL2;
 3675    sa1 = Y0 ^ Y2;
 3676    sa0 = Y1 ^ Y3;
 3677    sb1 = Y4 ^ Y6;
 3678    sb0 = Y5 ^ Y7;
 3679    ah = Y0 ^ Y1;
 3680    al = Y2 ^ Y3;
 3681    aa = sa0 ^ sa1;
 3682    bh = Y4 ^ Y5;
 3683    bl = Y6 ^ Y7;
 3684    bb = sb0 ^ sb1;
 3685    ab20 = sa0 ^ sb0;
 3686    ab22 = al ^ bl;
 3687    ab23 = Y3 ^ Y7;
 3688    ab21 = sa1 ^ sb1;
 3689    abcd1 = ah & bh;
 3690    rr1 = Y0 & Y4;
 3691    ph11 = ab20 ^ abcd1;
 3692    t01 = Y1 & Y5;
 3693    ph01 = t01 ^ abcd1;
 3694    abcd2 = al & bl;
 3695    r1 = Y2 & Y6;
 3696    pl11 = ab22 ^ abcd2;
 3697    r2 = Y3 & Y7;
 3698    pl01 = r2 ^ abcd2;
 3699    r3 = sa0 & sb0;
 3700    vr1 = aa & bb;
 3701    pr1 = vr1 ^ r3;
 3702    wr1 = sa1 & sb1;
 3703    qr1 = wr1 ^ r3;
 3704    ab0 = ph11 ^ rr1;
 3705    ab1 = ph01 ^ ab21;
 3706    ab2 = pl11 ^ r1;
 3707    ab3 = pl01 ^ qr1;
 3708    cp1 = ab0 ^ pr1;
 3709    cp2 = ab1 ^ qr1;
 3710    cp3 = ab2 ^ pr1;
 3711    cp4 = ab3 ^ ab23;
 3712    tinv1 = cp3 ^ cp4;
 3713    tinv2 = cp3 & cp1;
 3714    tinv3 = cp2 ^ tinv2;
 3715    tinv4 = cp1 ^ cp2;
 3716    tinv5 = cp4 ^ tinv2;
 3717    tinv6 = tinv5 & tinv4;
 3718    tinv7 = tinv3 & tinv1;
 3719    d2 = cp4 ^ tinv7;
 3720    d0 = cp2 ^ tinv6;
 3721    tinv8 = cp1 & cp4;
 3722    tinv9 = tinv4 & tinv8;
 3723    tinv10 = tinv4 ^ tinv2;
 3724    d1 = tinv9 ^ tinv10;
 3725    tinv11 = cp2 & cp3;
 3726    tinv12 = tinv1 & tinv11;
 3727    tinv13 = tinv1 ^ tinv2;
 3728    d3 = tinv12 ^ tinv13;
 3729    sd1 = d1 ^ d3;
 3730    sd0 = d0 ^ d2;
 3731    dl = d0 ^ d1;
 3732    dh = d2 ^ d3;
 3733    dd = sd0 ^ sd1;
 3734    abcd3 = dh & bh;
 3735    rr2 = d3 & Y4;
 3736    t02 = d2 & Y5;
 3737    abcd4 = dl & bl;
 3738    r4 = d1 & Y6;
 3739    r5 = d0 & Y7;
 3740    r6 = sd0 & sb0;
 3741    vr2 = dd & bb;
 3742    wr2 = sd1 & sb1;
 3743    abcd5 = dh & ah;
 3744    r7 = d3 & Y0;
 3745    r8 = d2 & Y1;
 3746    abcd6 = dl & al;
 3747    r9 = d1 & Y2;
 3748    r10 = d0 & Y3;
 3749    r11 = sd0 & sa0;
 3750    vr3 = dd & aa;
 3751    wr3 = sd1 & sa1;
 3752    ph12 = rr2 ^ abcd3;
 3753    ph02 = t02 ^ abcd3;
 3754    pl12 = r4 ^ abcd4;
 3755    pl02 = r5 ^ abcd4;
 3756    pr2 = vr2 ^ r6;
 3757    qr2 = wr2 ^ r6;
 3758    p0 = ph12 ^ pr2;
 3759    p1 = ph02 ^ qr2;
 3760    p2 = pl12 ^ pr2;
 3761    p3 = pl02 ^ qr2;
 3762    ph13 = r7 ^ abcd5;
 3763    ph03 = r8 ^ abcd5;
 3764    pl13 = r9 ^ abcd6;
 3765    pl03 = r10 ^ abcd6;
 3766    pr3 = vr3 ^ r11;
 3767    qr3 = wr3 ^ r11;
 3768    p4 = ph13 ^ pr3;
 3769    S7 = ph03 ^ qr3;
 3770    p6 = pl13 ^ pr3;
 3771    p7 = pl03 ^ qr3;
 3772    S3 = p1 ^ p6;
 3773    S6 = p2 ^ p6;
 3774    S0 = p3 ^ p6;
 3775    X11 = p0 ^ p2;
 3776    S5 = S0 ^ X11;
 3777    X13 = p4 ^ p7;
 3778    X14 = X11 ^ X13;
 3779    S1 = S3 ^ X14;
 3780    X16 = p1 ^ S7;
 3781    S2 = X14 ^ X16;
 3782    X18 = p0 ^ p4;
 3783    X19 = S5 ^ X16;
 3784    S4 = X18 ^ X19;
 3785
 3786    u[0] = S7;
 3787    u[1] = S6;
 3788    u[2] = S5;
 3789    u[3] = S4;
 3790    u[4] = S3;
 3791    u[5] = S2;
 3792    u[6] = S1;
 3793    u[7] = S0;
 3794}
 3795
 3796static void bs_inv_shift_rows(bs_word* b)
 3797{
 3798    bs_word t[AES_BLOCK_BITS];
 3799    int i;
 3800
 3801    for (i = 0; i < 128; i += 32) {
 3802        BS_ASSIGN_8(t, i +  0, b, (  0 + i) & BS_IDX_MASK);
 3803        BS_ASSIGN_8(t, i +  8, b, (104 + i) & BS_IDX_MASK);
 3804        BS_ASSIGN_8(t, i + 16, b, ( 80 + i) & BS_IDX_MASK);
 3805        BS_ASSIGN_8(t, i + 24, b, ( 56 + i) & BS_IDX_MASK);
 3806    }
 3807
 3808    XMEMCPY(b, t, sizeof(t));
 3809}
 3810
 3811#define O0  0
 3812#define O1  8
 3813#define O2  16
 3814#define O3  24
 3815
 3816#define BS_INV_MIX_SHIFT_8(br, b, O0, O1, O2, O3, of0, of1, of2)            \
 3817    of0 = b[O0+7] ^ b[O0+6] ^ b[O0+5] ^ b[O1 + 7] ^ b[O1+5] ^               \
 3818          b[O2+6] ^ b[O2+5] ^ b[O3+5];                                      \
 3819    of1 =           b[O0+7] ^ b[O0+6] ^             b[O1+6] ^               \
 3820          b[O2+7] ^ b[O2+6] ^ b[O3+6];                                      \
 3821    of2 =                     b[O0+7] ^             b[O1+7] ^               \
 3822                    b[O2+7] ^ b[O3+7];                                      \
 3823                                                                            \
 3824    br[0] =                                                   b[O1+0] ^     \
 3825            b[O2+0]                     ^ b[O3+0]           ^ of0;          \
 3826    br[1] = b[O0+0]                               ^ b[O1+0] ^ b[O1+1] ^     \
 3827            b[O2+1]                     ^ b[O3+1]           ^ of0 ^ of1;    \
 3828    br[2] = b[O0+1] ^ b[O0+0]                     ^ b[O1+1] ^ b[O1+2] ^     \
 3829            b[O2+2] ^ b[O2+0]           ^ b[O3+2]           ^ of1 ^ of2;    \
 3830    br[3] = b[O0+2] ^ b[O0+1] ^ b[O0+0] ^ b[O1+0] ^ b[O1+2] ^ b[O1+3] ^     \
 3831            b[O2+3] ^ b[O2+1] ^ b[O2+0] ^ b[O3+3] ^ b[O3+0] ^ of0 ^ of2;    \
 3832    br[4] = b[O0+3] ^ b[O0+2] ^ b[O0+1] ^ b[O1+1] ^ b[O1+3] ^ b[O1+4] ^     \
 3833            b[O2+4] ^ b[O2+2] ^ b[O2+1] ^ b[O3+4] ^ b[O3+1] ^ of0 ^ of1;    \
 3834    br[5] = b[O0+4] ^ b[O0+3] ^ b[O0+2] ^ b[O1+2] ^ b[O1+4] ^ b[O1+5] ^     \
 3835            b[O2+5] ^ b[O2+3] ^ b[O2+2] ^ b[O3+5] ^ b[O3+2] ^ of1 ^ of2;    \
 3836    br[6] = b[O0+5] ^ b[O0+4] ^ b[O0+3] ^ b[O1+3] ^ b[O1+5] ^ b[O1+6] ^     \
 3837            b[O2+6] ^ b[O2+4] ^ b[O2+3] ^ b[O3+6] ^ b[O3+3] ^ of2;          \
 3838    br[7] = b[O0+6] ^ b[O0+5] ^ b[O0+4] ^ b[O1+4] ^ b[O1+6] ^ b[O1+7] ^     \
 3839            b[O2+7] ^ b[O2+5] ^ b[O2+4] ^ b[O3+7] ^ b[O3+4]
 3840
 3841/* Inverse mix columns and shift rows. */
 3842static void bs_inv_mix_shift(bs_word* t, bs_word* b)
 3843{
 3844    bs_word* bp = b;
 3845    word8 or0 = BS_ROW_OFF_0 + BS_SHIFT_OFF_0;
 3846    word8 or1 = BS_ROW_OFF_1 + BS_SHIFT_OFF_1;
 3847    word8 or2 = BS_ROW_OFF_2 + BS_SHIFT_OFF_2;
 3848    word8 or3 = BS_ROW_OFF_3 + BS_SHIFT_OFF_3;
 3849    int i;
 3850
 3851    for (i = 0; i < AES_BLOCK_BITS / 4; i += AES_BLOCK_BITS / 16) {
 3852        bs_word* br;
 3853        bs_word of0;
 3854        bs_word of1;
 3855        bs_word of2;
 3856
 3857        br = t + or0;
 3858        BS_INV_MIX_SHIFT_8(br, bp, O0, O1, O2, O3, of0, of1, of2);
 3859        br = t + or1;
 3860        BS_INV_MIX_SHIFT_8(br, bp, O1, O2, O3, O0, of0, of1, of2);
 3861        br = t + or2;
 3862        BS_INV_MIX_SHIFT_8(br, bp, O2, O3, O0, O1, of0, of1, of2);
 3863        br = t + or3;
 3864        BS_INV_MIX_SHIFT_8(br, bp, O3, O0, O1, O2, of0, of1, of2);
 3865
 3866        or0 = (or0 + AES_BLOCK_BITS / 4) & BS_IDX_MASK;
 3867        or1 = (or1 + AES_BLOCK_BITS / 4) & BS_IDX_MASK;
 3868        or2 = (or2 + AES_BLOCK_BITS / 4) & BS_IDX_MASK;
 3869        or3 = (or3 + AES_BLOCK_BITS / 4) & BS_IDX_MASK;
 3870
 3871        bp += AES_BLOCK_BITS / 4;
 3872    }
 3873}
 3874
 3875static void bs_inv_sub_bytes_blocks(bs_word* b)
 3876{
 3877    int i;
 3878
 3879    for (i = 0; i < AES_BLOCK_BITS; i += 8) {
 3880        bs_inv_sub_bytes(b + i);
 3881    }
 3882}
 3883
 3884static void bs_decrypt(bs_word* state, bs_word* rk, word32 r)
 3885{
 3886    int i;
 3887    bs_word trans[AES_BLOCK_BITS];
 3888
 3889    bs_transpose(trans, state);
 3890
 3891    rk += r * AES_BLOCK_BITS;
 3892    bs_add_round_key(trans, trans, rk);
 3893    bs_inv_shift_rows(trans);
 3894    bs_inv_sub_bytes_blocks(trans);
 3895    rk -= AES_BLOCK_BITS;
 3896    bs_add_round_key(trans, trans, rk);
 3897    for (i = (int)r - 2; i >= 0; i--) {
 3898        bs_inv_mix_shift(state, trans);
 3899        bs_inv_sub_bytes_blocks(state);
 3900        rk -= AES_BLOCK_BITS;
 3901        bs_add_round_key(trans, state, rk);
 3902    }
 3903
 3904    bs_inv_transpose(state, trans);
 3905}
 3906
 3907#ifdef WOLFSSL_AES_DIRECT
 3908/* Decrypt a block using AES.
 3909 *
 3910 * @param [in]  aes       AES object.
 3911 * @param [in]  inBlock   Block to encrypt.
 3912 * @param [out] outBlock  Encrypted block.
 3913 * @param [in]  r         Rounds divided by 2.
 3914 */
 3915static void AesDecrypt_C(Aes* aes, const byte* inBlock, byte* outBlock,
 3916    word32 r)
 3917{
 3918    bs_word state[AES_BLOCK_BITS];
 3919
 3920    (void)r;
 3921
 3922    XMEMCPY(state, inBlock, WC_AES_BLOCK_SIZE);
 3923    XMEMSET(((byte*)state) + WC_AES_BLOCK_SIZE, 0, sizeof(state) - WC_AES_BLOCK_SIZE);
 3924
 3925    bs_decrypt(state, aes->bs_key, aes->rounds);
 3926
 3927    XMEMCPY(outBlock, state, WC_AES_BLOCK_SIZE);
 3928}
 3929#endif
 3930
 3931#if defined(HAVE_AES_ECB) && !(defined(WOLFSSL_IMX6_CAAM) && \
 3932    !defined(NO_IMX6_CAAM_AES) && !defined(WOLFSSL_QNX_CAAM))
 3933/* Decrypt a number of blocks using AES.
 3934 *
 3935 * @param [in]  aes  AES object.
 3936 * @param [in]  in   Block to encrypt.
 3937 * @param [out] out  Encrypted block.
 3938 * @param [in]  sz   Number of blocks to encrypt.
 3939 */
 3940static void AesDecryptBlocks_C(Aes* aes, const byte* in, byte* out, word32 sz)
 3941{
 3942    bs_word state[AES_BLOCK_BITS];
 3943
 3944    while (sz >= BS_BLOCK_SIZE) {
 3945        XMEMCPY(state, in, BS_BLOCK_SIZE);
 3946        bs_decrypt(state, aes->bs_key, aes->rounds);
 3947        XMEMCPY(out, state, BS_BLOCK_SIZE);
 3948        sz  -= BS_BLOCK_SIZE;
 3949        in  += BS_BLOCK_SIZE;
 3950        out += BS_BLOCK_SIZE;
 3951    }
 3952    if (sz > 0) {
 3953        XMEMCPY(state, in, sz);
 3954        XMEMSET(((byte*)state) + sz, 0, sizeof(state) - sz);
 3955        bs_decrypt(state, aes->bs_key, aes->rounds);
 3956        XMEMCPY(out, state, sz);
 3957    }
 3958}
 3959#endif
 3960
 3961#endif /* !WC_AES_BITSLICED */
 3962#endif
 3963
 3964#if (defined(HAVE_AES_CBC) && !defined(WOLFSSL_DEVCRYPTO_CBC)) || \
 3965    defined(WOLFSSL_AES_DIRECT)
 3966#if defined(__aarch64__) || !defined(WOLFSSL_ARMASM)
 3967#if !defined(WC_AES_BITSLICED) || defined(WOLFSSL_AES_DIRECT)
 3968/* Software AES - ECB Decrypt */
 3969
 3970#ifdef WC_AES_HAVE_PREFETCH_ARG
 3971#define wc_AesDecrypt(aes, inBlock, outBlock) \
 3972    AesDecrypt_preFetchOpt(aes, inBlock, outBlock, &always_prefetch)
 3973WC_ALL_ARGS_NOT_NULL static WARN_UNUSED_RESULT int AesDecrypt_preFetchOpt(
 3974    Aes* aes, const byte* inBlock, byte* outBlock, int *prefetch_ptr)
 3975#else
 3976#define AesDecrypt_preFetchOpt(aes, inBlock, outBlock, prefetch_ptr) \
 3977    wc_AesDecrypt(aes, inBlock, outBlock)
 3978WC_ALL_ARGS_NOT_NULL static WARN_UNUSED_RESULT int wc_AesDecrypt(
 3979    Aes* aes, const byte* inBlock, byte* outBlock)
 3980#endif
 3981{
 3982#if defined(MAX3266X_AES)
 3983    word32 keySize;
 3984#endif
 3985#if defined(MAX3266X_CB)
 3986    int ret_cb;
 3987#endif
 3988    word32 r;
 3989
 3990#ifdef WC_DEBUG_CIPHER_LIFECYCLE
 3991    {
 3992        int ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
 3993        if (ret < 0)
 3994            return ret;
 3995    }
 3996#endif
 3997
 3998    r = aes->rounds >> 1;
 3999
 4000    if (r > 7 || r == 0) {
 4001        WOLFSSL_ERROR_VERBOSE(KEYUSAGE_E);
 4002        return KEYUSAGE_E;
 4003    }
 4004
 4005#ifdef WOLFSSL_AESNI
 4006    if (aes->use_aesni) {
 4007        ASSERT_SAVED_VECTOR_REGISTERS();
 4008
 4009        #ifdef DEBUG_AESNI
 4010            printf("about to aes decrypt\n");
 4011            printf("in  = %p\n", inBlock);
 4012            printf("out = %p\n", outBlock);
 4013            printf("aes->key = %p\n", aes->key);
 4014            printf("aes->rounds = %d\n", aes->rounds);
 4015            printf("sz = %d\n", WC_AES_BLOCK_SIZE);
 4016        #endif
 4017
 4018        /* if input and output same will overwrite input iv */
 4019        if ((const byte*)aes->tmp != inBlock)
 4020            XMEMCPY(aes->tmp, inBlock, WC_AES_BLOCK_SIZE);
 4021        AES_ECB_decrypt_AESNI(inBlock, outBlock, WC_AES_BLOCK_SIZE, (byte*)aes->key,
 4022                        (int)aes->rounds);
 4023        return 0;
 4024    }
 4025    else {
 4026        #ifdef DEBUG_AESNI
 4027            printf("Skipping AES-NI\n");
 4028        #endif
 4029    }
 4030#elif defined(WOLFSSL_ARMASM)
 4031#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
 4032#if !defined(__aarch64__)
 4033    AES_decrypt_AARCH32(inBlock, outBlock, (byte*)aes->key, (int)aes->rounds);
 4034#else
 4035    if (aes->use_aes_hw_crypto) {
 4036        AES_decrypt_AARCH64(inBlock, outBlock, (byte*)aes->key,
 4037            (int)aes->rounds);
 4038    }
 4039    else
 4040#endif /* !__aarch64__ */
 4041#endif /* !WOLFSSL_ARMASM_NO_HW_CRYPTO */
 4042#if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON) && \
 4043    defined(WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP)
 4044    {
 4045        AES_ECB_decrypt_NEON(inBlock, outBlock, WC_AES_BLOCK_SIZE,
 4046            (const unsigned char*)aes->key, aes->rounds);
 4047    }
 4048#elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
 4049    {
 4050        AES_ECB_decrypt(inBlock, outBlock, WC_AES_BLOCK_SIZE,
 4051            (const unsigned char*)aes->key, aes->rounds);
 4052    }
 4053#endif
 4054    return 0;
 4055#endif /* WOLFSSL_AESNI */
 4056#if defined(WOLFSSL_SCE) && !defined(WOLFSSL_SCE_NO_AES)
 4057    return AES_ECB_decrypt(aes, inBlock, outBlock, WC_AES_BLOCK_SIZE);
 4058#endif
 4059#if defined(WOLFSSL_IMXRT_DCP)
 4060    if (aes->keylen == 16) {
 4061        DCPAesEcbDecrypt(aes, outBlock, inBlock, WC_AES_BLOCK_SIZE);
 4062        return 0;
 4063    }
 4064#endif
 4065#if defined(WOLFSSL_SE050) && defined(WOLFSSL_SE050_CRYPT)
 4066    if (aes->useSWCrypt == 0) {
 4067        return se050_aes_crypt(aes, inBlock, outBlock, WC_AES_BLOCK_SIZE,
 4068                               AES_DECRYPTION, kAlgorithm_SSS_AES_ECB);
 4069    }
 4070#endif
 4071#if defined(WOLFSSL_ESPIDF) && defined(NEED_AES_HW_FALLBACK)
 4072    if (wc_esp32AesSupportedKeyLen(aes)) {
 4073        return wc_esp32AesDecrypt(aes, inBlock, outBlock);
 4074    }
 4075    else {
 4076        /* For example, the ESP32-S3 does not support HW for len = 24,
 4077         * so fall back to SW */
 4078    #ifdef DEBUG_WOLFSSL
 4079        ESP_LOGW(TAG, "wc_AesDecrypt HW Falling back, "
 4080                        "unsupported keylen = %d", aes->keylen);
 4081    #endif
 4082    } /* else !wc_esp32AesSupportedKeyLen for ESP32 */
 4083#endif
 4084
 4085#if defined(MAX3266X_AES)
 4086    if (wc_AesGetKeySize(aes, &keySize) == 0) {
 4087        return wc_MXC_TPU_AesDecrypt(inBlock, (byte*)aes->reg, (byte*)aes->key,
 4088                                    MXC_TPU_MODE_ECB, WC_AES_BLOCK_SIZE,
 4089                                    outBlock, (unsigned int)keySize);
 4090    }
 4091#endif
 4092
 4093#if defined(MAX3266X_CB) && defined(HAVE_AES_ECB) /* Can do a basic ECB block */
 4094    #ifndef WOLF_CRYPTO_CB_FIND
 4095    if (aes->devId != INVALID_DEVID)
 4096    #endif
 4097    {
 4098        ret_cb = wc_CryptoCb_AesEcbDecrypt(aes, outBlock, inBlock,
 4099                                            WC_AES_BLOCK_SIZE);
 4100        if (ret_cb != WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE))
 4101            return ret_cb;
 4102        /* fall-through when unavailable */
 4103    }
 4104#endif
 4105
 4106#ifdef WC_AES_HAVE_PREFETCH_ARG
 4107    AesDecrypt_C(aes, inBlock, outBlock, r, prefetch_ptr);
 4108#else
 4109    AesDecrypt_C(aes, inBlock, outBlock, r);
 4110#endif
 4111
 4112    return 0;
 4113} /* wc_AesDecrypt[_SW]() */
 4114#endif /* !WC_AES_BITSLICED || WOLFSSL_AES_DIRECT */
 4115#endif
 4116#endif /* HAVE_AES_CBC || WOLFSSL_AES_DIRECT */
 4117#endif /* HAVE_AES_DECRYPT */
 4118
 4119#endif /* NEED_AES_TABLES */
 4120
 4121#ifdef WOLF_CRYPTO_CB_ONLY_AES
 4122/* Under WOLF_CRYPTO_CB_ONLY_AES the per-block primitive is a thin shim over
 4123 * the cryptocb ECB callback. When the callback returns CRYPTOCB_UNAVAILABLE
 4124 * there is no software fallback, so the operation fails with NO_VALID_DEVID. */
 4125static WARN_UNUSED_RESULT int wc_AesEncrypt(Aes* aes, const byte* inBlock,
 4126    byte* outBlock)
 4127{
 4128    int ret;
 4129
 4130    if (aes == NULL || inBlock == NULL || outBlock == NULL)
 4131        return BAD_FUNC_ARG;
 4132#ifdef WC_DEBUG_CIPHER_LIFECYCLE
 4133    ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
 4134    if (ret < 0)
 4135        return ret;
 4136#endif
 4137
 4138    ret = wc_CryptoCb_AesEcbEncrypt(aes, outBlock, inBlock, WC_AES_BLOCK_SIZE);
 4139    if (ret != WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE))
 4140        return ret;
 4141    return NO_VALID_DEVID;
 4142}
 4143
 4144#ifdef HAVE_AES_DECRYPT
 4145static WARN_UNUSED_RESULT int wc_AesDecrypt(Aes* aes, const byte* inBlock,
 4146    byte* outBlock)
 4147{
 4148    int ret;
 4149
 4150    if (aes == NULL || inBlock == NULL || outBlock == NULL)
 4151        return BAD_FUNC_ARG;
 4152#ifdef WC_DEBUG_CIPHER_LIFECYCLE
 4153    ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
 4154    if (ret < 0)
 4155        return ret;
 4156#endif
 4157
 4158    ret = wc_CryptoCb_AesEcbDecrypt(aes, outBlock, inBlock, WC_AES_BLOCK_SIZE);
 4159    if (ret != WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE))
 4160        return ret;
 4161    return NO_VALID_DEVID;
 4162}
 4163#endif /* HAVE_AES_DECRYPT */
 4164#endif /* WOLF_CRYPTO_CB_ONLY_AES */
 4165
 4166#ifndef WC_AES_HAVE_PREFETCH_ARG
 4167    #ifndef AesEncrypt_preFetchOpt
 4168        #define AesEncrypt_preFetchOpt(aes, inBlock, outBlock, do_preFetch) \
 4169            wc_AesEncrypt(aes, inBlock, outBlock)
 4170    #endif
 4171    #ifndef AesDecrypt_preFetchOpt
 4172        #define AesDecrypt_preFetchOpt(aes, inBlock, outBlock, do_preFetch) \
 4173            wc_AesDecrypt(aes, inBlock, outBlock)
 4174    #endif
 4175#endif
 4176
 4177/* wc_AesSetKey */
 4178#if defined(STM32_CRYPTO)
 4179
 4180    int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen,
 4181            const byte* iv, int dir)
 4182    {
 4183        word32 *rk;
 4184
 4185        (void)dir;
 4186
 4187        if (aes == NULL || (keylen != 16 &&
 4188        #ifdef WOLFSSL_AES_192
 4189            keylen != 24 &&
 4190        #endif
 4191            keylen != 32)) {
 4192            return BAD_FUNC_ARG;
 4193        }
 4194
 4195#ifdef WC_DEBUG_CIPHER_LIFECYCLE
 4196        {
 4197            int ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
 4198            if (ret < 0)
 4199                return ret;
 4200        }
 4201#endif
 4202
 4203        rk = aes->key;
 4204        aes->keylen = keylen;
 4205        aes->rounds = keylen/4 + 6;
 4206        XMEMCPY(rk, userKey, keylen);
 4207    #if !defined(WOLFSSL_STM32_CUBEMX) || defined(STM32_HAL_V2)
 4208        ByteReverseWords(rk, rk, keylen);
 4209    #endif
 4210    #if defined(WOLFSSL_AES_COUNTER) || defined(WOLFSSL_AES_CFB) || \
 4211        defined(WOLFSSL_AES_OFB) || defined(WOLFSSL_AES_XTS) || \
 4212        defined(WOLFSSL_AES_CTS)
 4213        aes->left = 0;
 4214    #endif
 4215        return wc_AesSetIV(aes, iv);
 4216    }
 4217    #if defined(WOLFSSL_AES_DIRECT)
 4218        int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
 4219                            const byte* iv, int dir)
 4220        {
 4221            return wc_AesSetKey(aes, userKey, keylen, iv, dir);
 4222        }
 4223    #endif
 4224
 4225#elif defined(HAVE_COLDFIRE_SEC)
 4226    #if defined (HAVE_THREADX)
 4227        #include "memory_pools.h"
 4228        extern TX_BYTE_POOL mp_ncached;  /* Non Cached memory pool */
 4229    #endif
 4230
 4231    #define AES_BUFFER_SIZE (WC_AES_BLOCK_SIZE * 64)
 4232    static unsigned char *AESBuffIn = NULL;
 4233    static unsigned char *AESBuffOut = NULL;
 4234    static byte *secReg;
 4235    static byte *secKey;
 4236    static volatile SECdescriptorType *secDesc;
 4237
 4238    static wolfSSL_Mutex Mutex_AesSEC;
 4239
 4240    #define SEC_DESC_AES_CBC_ENCRYPT 0x60300010
 4241    #define SEC_DESC_AES_CBC_DECRYPT 0x60200010
 4242
 4243    extern volatile unsigned char __MBAR[];
 4244
 4245    int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen,
 4246        const byte* iv, int dir)
 4247    {
 4248        if (AESBuffIn == NULL) {
 4249        #if defined (HAVE_THREADX)
 4250            int s1, s2, s3, s4, s5;
 4251            s5 = tx_byte_allocate(&mp_ncached,(void *)&secDesc,
 4252                                  sizeof(SECdescriptorType), TX_NO_WAIT);
 4253            s1 = tx_byte_allocate(&mp_ncached, (void *)&AESBuffIn,
 4254                                  AES_BUFFER_SIZE, TX_NO_WAIT);
 4255            s2 = tx_byte_allocate(&mp_ncached, (void *)&AESBuffOut,
 4256                                  AES_BUFFER_SIZE, TX_NO_WAIT);
 4257            s3 = tx_byte_allocate(&mp_ncached, (void *)&secKey,
 4258                                  WC_AES_BLOCK_SIZE*2, TX_NO_WAIT);
 4259            s4 = tx_byte_allocate(&mp_ncached, (void *)&secReg,
 4260                                  WC_AES_BLOCK_SIZE, TX_NO_WAIT);
 4261
 4262            if (s1 || s2 || s3 || s4 || s5)
 4263                return BAD_FUNC_ARG;
 4264        #else
 4265            #warning "Allocate non-Cache buffers"
 4266        #endif
 4267
 4268            wc_InitMutex(&Mutex_AesSEC);
 4269        }
 4270
 4271        if (!((keylen == 16) || (keylen == 24) || (keylen == 32)))
 4272            return BAD_FUNC_ARG;
 4273
 4274        if (aes == NULL)
 4275            return BAD_FUNC_ARG;
 4276
 4277#ifdef WC_DEBUG_CIPHER_LIFECYCLE
 4278        {
 4279            int ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
 4280            if (ret < 0)
 4281                return ret;
 4282        }
 4283#endif
 4284
 4285        aes->keylen = keylen;
 4286        aes->rounds = keylen/4 + 6;
 4287        XMEMCPY(aes->key, userKey, keylen);
 4288
 4289        if (iv)
 4290            XMEMCPY(aes->reg, iv, WC_AES_BLOCK_SIZE);
 4291
 4292    #if defined(WOLFSSL_AES_COUNTER) || defined(WOLFSSL_AES_CFB) || \
 4293        defined(WOLFSSL_AES_OFB) || defined(WOLFSSL_AES_XTS) || \
 4294        defined(WOLFSSL_AES_CTS)
 4295        aes->left = 0;
 4296    #endif
 4297
 4298        return 0;
 4299    }
 4300#elif defined(FREESCALE_LTC)
 4301    int wc_AesSetKeyLocal(Aes* aes, const byte* userKey, word32 keylen,
 4302        const byte* iv, int dir, int checkKeyLen)
 4303    {
 4304        if (aes == NULL)
 4305            return BAD_FUNC_ARG;
 4306
 4307#ifdef WC_DEBUG_CIPHER_LIFECYCLE
 4308        {
 4309            int ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
 4310            if (ret < 0)
 4311                return ret;
 4312        }
 4313#endif
 4314
 4315        if (checkKeyLen) {
 4316            if (!((keylen == 16) || (keylen == 24) || (keylen == 32)))
 4317                return BAD_FUNC_ARG;
 4318        }
 4319        (void)dir;
 4320
 4321        aes->rounds = keylen/4 + 6;
 4322        XMEMCPY(aes->key, userKey, keylen);
 4323
 4324    #if defined(WOLFSSL_AES_COUNTER) || defined(WOLFSSL_AES_CFB) || \
 4325        defined(WOLFSSL_AES_OFB) || defined(WOLFSSL_AES_XTS) || \
 4326        defined(WOLFSSL_AES_CTS)
 4327        aes->left = 0;
 4328    #endif
 4329
 4330        return wc_AesSetIV(aes, iv);
 4331    }
 4332
 4333    int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen,
 4334        const byte* iv, int dir)
 4335    {
 4336        if (aes == NULL || userKey == NULL) {
 4337            return BAD_FUNC_ARG;
 4338        }
 4339        if (keylen > sizeof(aes->key)) {
 4340            return BAD_FUNC_ARG;
 4341        }
 4342
 4343        return wc_AesSetKeyLocal(aes, userKey, keylen, iv, dir, 1);
 4344    }
 4345
 4346    int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
 4347                        const byte* iv, int dir)
 4348    {
 4349        return wc_AesSetKey(aes, userKey, keylen, iv, dir);
 4350    }
 4351#elif defined(WOLFSSL_NRF51_AES)
 4352    int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen,
 4353        const byte* iv, int dir)
 4354    {
 4355        int ret;
 4356
 4357        (void)dir;
 4358        (void)iv;
 4359
 4360        if (aes == NULL || keylen != 16)
 4361            return BAD_FUNC_ARG;
 4362
 4363#ifdef WC_DEBUG_CIPHER_LIFECYCLE
 4364        ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
 4365        if (ret < 0)
 4366            return ret;
 4367#endif
 4368
 4369        aes->keylen = keylen;
 4370        aes->rounds = keylen/4 + 6;
 4371        XMEMCPY(aes->key, userKey, keylen);
 4372        ret = nrf51_aes_set_key(userKey);
 4373
 4374    #if defined(WOLFSSL_AES_COUNTER) || defined(WOLFSSL_AES_CFB) || \
 4375        defined(WOLFSSL_AES_OFB) || defined(WOLFSSL_AES_XTS) || \
 4376        defined(WOLFSSL_AES_CTS)
 4377        aes->left = 0;
 4378    #endif
 4379
 4380        return ret;
 4381    }
 4382
 4383    int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
 4384                        const byte* iv, int dir)
 4385    {
 4386        return wc_AesSetKey(aes, userKey, keylen, iv, dir);
 4387    }
 4388#elif defined(WOLFSSL_ESP32_CRYPT) && !defined(NO_WOLFSSL_ESP32_CRYPT_AES)
 4389    /* This is the only definition for HW only.
 4390     * but needs to be renamed when fallback needed.
 4391     * See call in wc_AesSetKey() */
 4392    int wc_AesSetKey_for_ESP32(Aes* aes, const byte* userKey, word32 keylen,
 4393        const byte* iv, int dir)
 4394    {
 4395        (void)dir;
 4396        (void)iv;
 4397        ESP_LOGV(TAG, "wc_AesSetKey_for_ESP32");
 4398        if (aes == NULL || (keylen != 16 && keylen != 24 && keylen != 32)) {
 4399            return BAD_FUNC_ARG;
 4400        }
 4401
 4402#ifdef WC_DEBUG_CIPHER_LIFECYCLE
 4403        {
 4404            int ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
 4405            if (ret < 0)
 4406                return ret;
 4407        }
 4408#endif
 4409
 4410    #if !defined(WOLFSSL_AES_128)
 4411        if (keylen == 16) {
 4412            return BAD_FUNC_ARG;
 4413        }
 4414    #endif
 4415
 4416    #if !defined(WOLFSSL_AES_192)
 4417        if (keylen == 24) {
 4418            return BAD_FUNC_ARG;
 4419        }
 4420    #endif
 4421
 4422    #if !defined(WOLFSSL_AES_256)
 4423        if (keylen == 32) {
 4424            return BAD_FUNC_ARG;
 4425        }
 4426    #endif
 4427
 4428        aes->keylen = keylen;
 4429        aes->rounds = keylen/4 + 6;
 4430
 4431        XMEMCPY(aes->key, userKey, keylen);
 4432        #if defined(WOLFSSL_AES_COUNTER) || defined(WOLFSSL_AES_CFB) || \
 4433            defined(WOLFSSL_AES_OFB) || defined(WOLFSSL_AES_XTS) || \
 4434            defined(WOLFSSL_AES_CTS)
 4435            aes->left = 0;
 4436        #endif
 4437        return wc_AesSetIV(aes, iv);
 4438    } /* wc_AesSetKey */
 4439
 4440    /* end #elif ESP32 */
 4441#elif defined(WOLFSSL_CRYPTOCELL) && defined(WOLFSSL_CRYPTOCELL_AES)
 4442
 4443    int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen, const byte* iv,
 4444                    int dir)
 4445    {
 4446        SaSiError_t ret = SASI_OK;
 4447        SaSiAesIv_t iv_aes;
 4448
 4449        if (aes == NULL ||
 4450           (keylen != AES_128_KEY_SIZE &&
 4451            keylen != AES_192_KEY_SIZE &&
 4452            keylen != AES_256_KEY_SIZE)) {
 4453            return BAD_FUNC_ARG;
 4454        }
 4455
 4456#ifdef WC_DEBUG_CIPHER_LIFECYCLE
 4457        {
 4458            int ret2 =
 4459                wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
 4460            if (ret2 < 0)
 4461                return ret2;
 4462        }
 4463#endif
 4464
 4465    #if defined(AES_MAX_KEY_SIZE)
 4466        if (keylen > (AES_MAX_KEY_SIZE/8)) {
 4467            return BAD_FUNC_ARG;
 4468        }
 4469    #endif
 4470        if (dir != AES_ENCRYPTION &&
 4471            dir != AES_DECRYPTION) {
 4472            return BAD_FUNC_ARG;
 4473        }
 4474
 4475        if (dir == AES_ENCRYPTION) {
 4476            aes->ctx.mode = SASI_AES_ENCRYPT;
 4477            SaSi_AesInit(&aes->ctx.user_ctx,
 4478                         SASI_AES_ENCRYPT,
 4479                         SASI_AES_MODE_CBC,
 4480                         SASI_AES_PADDING_NONE);
 4481        }
 4482        else {
 4483            aes->ctx.mode = SASI_AES_DECRYPT;
 4484            SaSi_AesInit(&aes->ctx.user_ctx,
 4485                         SASI_AES_DECRYPT,
 4486                         SASI_AES_MODE_CBC,
 4487                         SASI_AES_PADDING_NONE);
 4488        }
 4489
 4490        aes->keylen = keylen;
 4491        aes->rounds = keylen/4 + 6;
 4492        XMEMCPY(aes->key, userKey, keylen);
 4493
 4494        aes->ctx.key.pKey = (byte*)aes->key;
 4495        aes->ctx.key.keySize= keylen;
 4496
 4497        ret = SaSi_AesSetKey(&aes->ctx.user_ctx,
 4498                             SASI_AES_USER_KEY,
 4499                             &aes->ctx.key,
 4500                             sizeof(aes->ctx.key));
 4501        if (ret != SASI_OK) {
 4502            return BAD_FUNC_ARG;
 4503        }
 4504
 4505        ret = wc_AesSetIV(aes, iv);
 4506
 4507        if (iv)
 4508            XMEMCPY(iv_aes, iv, WC_AES_BLOCK_SIZE);
 4509        else
 4510            XMEMSET(iv_aes,  0, WC_AES_BLOCK_SIZE);
 4511
 4512
 4513        ret = SaSi_AesSetIv(&aes->ctx.user_ctx, iv_aes);
 4514        if (ret != SASI_OK) {
 4515            return ret;
 4516        }
 4517       return ret;
 4518    }
 4519    #if defined(WOLFSSL_AES_DIRECT)
 4520        int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
 4521                            const byte* iv, int dir)
 4522        {
 4523            return wc_AesSetKey(aes, userKey, keylen, iv, dir);
 4524        }
 4525    #endif
 4526
 4527#elif defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES) \
 4528    && !defined(WOLFSSL_QNX_CAAM)
 4529      /* implemented in wolfcrypt/src/port/caam/caam_aes.c */
 4530
 4531#elif defined(WOLFSSL_AFALG)
 4532    /* implemented in wolfcrypt/src/port/af_alg/afalg_aes.c */
 4533
 4534#elif defined(WOLFSSL_DEVCRYPTO_AES)
 4535    /* implemented in wolfcrypt/src/port/devcrypto/devcrypto_aes.c */
 4536
 4537#elif defined(WOLFSSL_SILABS_SE_ACCEL)
 4538    /* implemented in wolfcrypt/src/port/silabs/silabs_aes.c */
 4539
 4540#elif defined(WOLFSSL_RENESAS_FSPSM_CRYPTONLY) && \
 4541     !defined(NO_WOLFSSL_RENESAS_FSPSM_AES)
 4542    /* implemented in wolfcrypt/src/port/renesas/renesas_fspsm_aes.c */
 4543
 4544#elif !defined(__aarch64__) && defined(WOLFSSL_ARMASM)
 4545    static int AesSetKey(Aes* aes, const byte* userKey, word32 keylen,
 4546            const byte* iv, int dir)
 4547    {
 4548    #if defined(WOLFSSL_AES_COUNTER) || defined(WOLFSSL_AES_CFB) || \
 4549        defined(WOLFSSL_AES_OFB) || defined(WOLFSSL_AES_XTS) || \
 4550        defined(WOLFSSL_AES_CTS)
 4551        aes->left = 0;
 4552    #endif
 4553
 4554        aes->keylen = (int)keylen;
 4555        aes->rounds = (keylen/4) + 6;
 4556
 4557#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
 4558        AES_set_key_AARCH32(userKey, keylen, (byte*)aes->key, dir);
 4559#else
 4560        AES_set_encrypt_key(userKey, keylen * 8, (byte*)aes->key);
 4561
 4562    #ifdef HAVE_AES_DECRYPT
 4563        if (dir == AES_DECRYPTION) {
 4564            AES_invert_key((byte*)aes->key, aes->rounds);
 4565        }
 4566    #else
 4567        (void)dir;
 4568    #endif
 4569#endif
 4570        return wc_AesSetIV(aes, iv);
 4571    }
 4572
 4573    int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen,
 4574            const byte* iv, int dir)
 4575    {
 4576#if defined(WOLF_CRYPTO_CB) && defined(WOLF_CRYPTO_CB_SETKEY)
 4577        int cbRet;
 4578#endif
 4579        if ((aes == NULL) || (userKey == NULL)) {
 4580            return BAD_FUNC_ARG;
 4581        }
 4582
 4583        switch (keylen) {
 4584    #if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 128 &&     \
 4585        defined(WOLFSSL_AES_128)
 4586        case 16:
 4587    #endif
 4588    #if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 192 &&     \
 4589        defined(WOLFSSL_AES_192)
 4590        case 24:
 4591    #endif
 4592    #if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 256 &&     \
 4593        defined(WOLFSSL_AES_256)
 4594        case 32:
 4595    #endif
 4596            break;
 4597        default:
 4598            return BAD_FUNC_ARG;
 4599        }
 4600
 4601    #ifdef WOLF_CRYPTO_CB
 4602        if (aes->devId != INVALID_DEVID) {
 4603        #ifdef WOLF_CRYPTO_CB_AES_SETKEY
 4604            int ret = wc_CryptoCb_AesSetKey(aes, userKey, keylen);
 4605            if (ret == 0) {
 4606                /* Callback succeeded - SE owns the key */
 4607                aes->keylen = (int)keylen;
 4608                if (iv != NULL)
 4609                    XMEMCPY(aes->reg, iv, WC_AES_BLOCK_SIZE);
 4610                else
 4611                    XMEMSET(aes->reg, 0, WC_AES_BLOCK_SIZE);
 4612                return 0;
 4613            }
 4614            else if (ret != WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE)) {
 4615                aes->devCtx = NULL;
 4616                return ret;
 4617            }
 4618            /* CRYPTOCB_UNAVAILABLE: continue to software setup */
 4619        #endif
 4620        #ifdef WOLF_CRYPTO_CB_SETKEY
 4621            cbRet = wc_CryptoCb_SetKey(aes->devId,
 4622                WC_SETKEY_AES, aes, (void*)userKey, keylen,
 4623                (void*)iv,
 4624                (iv != NULL) ? WC_AES_BLOCK_SIZE : 0, dir);
 4625            if (cbRet != WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE))
 4626                return cbRet;
 4627            /* CRYPTOCB_UNAVAILABLE: fall through to software setup */
 4628        #endif /* WOLF_CRYPTO_CB_SETKEY */
 4629            /* Standard CryptoCB path - copy key to devKey for encrypt/decrypt offload */
 4630            if (keylen > sizeof(aes->devKey)) {
 4631                return BAD_FUNC_ARG;
 4632            }
 4633            XMEMCPY(aes->devKey, userKey, keylen);
 4634        }
 4635    #endif
 4636
 4637        return AesSetKey(aes, userKey, keylen, iv, dir);
 4638    }
 4639
 4640    #if defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER)
 4641        /* AES-CTR and AES-DIRECT need to use this for key setup */
 4642        /* This function allows key sizes that are not 128/192/256 bits */
 4643    int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
 4644                           const byte* iv, int dir)
 4645    {
 4646        if (aes == NULL) {
 4647            return BAD_FUNC_ARG;
 4648        }
 4649        if (keylen > sizeof(aes->key)) {
 4650            return BAD_FUNC_ARG;
 4651        }
 4652
 4653        return AesSetKey(aes, userKey, keylen, iv, dir);
 4654    }
 4655    #endif /* WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER */
 4656#elif defined(FREESCALE_MMCAU)
 4657    int wc_AesSetKeyLocal(Aes* aes, const byte* userKey, word32 keylen,
 4658        const byte* iv, int dir, int checkKeyLen)
 4659    {
 4660        int ret;
 4661        byte* rk;
 4662        byte* tmpKey = (byte*)userKey;
 4663        int tmpKeyDynamic = 0;
 4664        word32 alignOffset = 0;
 4665
 4666        (void)dir;
 4667
 4668        if (aes == NULL)
 4669            return BAD_FUNC_ARG;
 4670
 4671#ifdef WC_DEBUG_CIPHER_LIFECYCLE
 4672        {
 4673            int ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
 4674            if (ret < 0)
 4675                return ret;
 4676        }
 4677#endif
 4678
 4679        if (checkKeyLen) {
 4680            if (!((keylen == 16) || (keylen == 24) || (keylen == 32)))
 4681                return BAD_FUNC_ARG;
 4682        }
 4683
 4684        rk = (byte*)aes->key;
 4685        if (rk == NULL)
 4686            return BAD_FUNC_ARG;
 4687
 4688    #if defined(WOLFSSL_AES_COUNTER) || defined(WOLFSSL_AES_CFB) || \
 4689        defined(WOLFSSL_AES_OFB) || defined(WOLFSSL_AES_XTS) || \
 4690        defined(WOLFSSL_AES_CTS)
 4691        aes->left = 0;
 4692    #endif
 4693
 4694        aes->rounds = keylen/4 + 6;
 4695
 4696    #ifdef FREESCALE_MMCAU_CLASSIC
 4697        if ((wc_ptr_t)userKey % WOLFSSL_MMCAU_ALIGNMENT) {
 4698        #ifndef NO_WOLFSSL_ALLOC_ALIGN
 4699            byte* tmp = (byte*)XMALLOC(keylen + WOLFSSL_MMCAU_ALIGNMENT,
 4700                                       aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
 4701            if (tmp == NULL) {
 4702                return MEMORY_E;
 4703            }
 4704            alignOffset = WOLFSSL_MMCAU_ALIGNMENT -
 4705                          ((wc_ptr_t)tmp % WOLFSSL_MMCAU_ALIGNMENT);
 4706            tmpKey = tmp + alignOffset;
 4707            XMEMCPY(tmpKey, userKey, keylen);
 4708            tmpKeyDynamic = 1;
 4709        #else
 4710            WOLFSSL_MSG("Bad cau_aes_set_key alignment");
 4711            return BAD_ALIGN_E;
 4712        #endif
 4713        }
 4714    #endif
 4715
 4716        ret = wolfSSL_CryptHwMutexLock();
 4717        if(ret == 0) {
 4718        #ifdef FREESCALE_MMCAU_CLASSIC
 4719            cau_aes_set_key(tmpKey, keylen*8, rk);
 4720        #else
 4721            MMCAU_AES_SetKey(tmpKey, keylen, rk);
 4722        #endif
 4723            wolfSSL_CryptHwMutexUnLock();
 4724
 4725            ret = wc_AesSetIV(aes, iv);
 4726        }
 4727
 4728        if (tmpKeyDynamic == 1) {
 4729            XFREE(tmpKey - alignOffset, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
 4730        }
 4731
 4732        return ret;
 4733    }
 4734
 4735    int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen,
 4736        const byte* iv, int dir)
 4737    {
 4738        return wc_AesSetKeyLocal(aes, userKey, keylen, iv, dir, 1);
 4739    }
 4740
 4741    int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
 4742                        const byte* iv, int dir)
 4743    {
 4744        return wc_AesSetKey(aes, userKey, keylen, iv, dir);
 4745    }
 4746
 4747#elif defined(WOLFSSL_PSOC6_CRYPTO)
 4748
 4749    int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen,
 4750        const byte* iv, int dir)
 4751    {
 4752        return wc_Psoc6_Aes_SetKey(aes, userKey, keylen, iv, dir);
 4753    }
 4754
 4755    #if defined(WOLFSSL_AES_DIRECT)
 4756        int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
 4757                            const byte* iv, int dir)
 4758        {
 4759            return wc_AesSetKey(aes, userKey, keylen, iv, dir);
 4760        }
 4761    #endif /* WOLFSSL_AES_DIRECT */
 4762#else
 4763    #define NEED_SOFTWARE_AES_SETKEY
 4764#endif
 4765
 4766/* Either we fell though with no HW support at all,
 4767 * or perhaps there's HW support for *some* keylengths
 4768 * and we need both HW and SW. */
 4769#ifdef NEED_SOFTWARE_AES_SETKEY
 4770
 4771#ifdef NEED_AES_TABLES
 4772
 4773#ifndef WC_AES_BITSLICED
 4774#if !defined(WOLFSSL_ARMASM)
 4775/* Set the AES key and expand.
 4776 *
 4777 * @param [in]  aes    AES object.
 4778 * @param [in]  key    Block to encrypt.
 4779 * @param [in]  keySz  Number of bytes in key.
 4780 * @param [in]  dir    Direction of crypt: AES_ENCRYPTION or AES_DECRYPTION.
 4781 */
 4782static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir)
 4783{
 4784#ifdef WC_C_DYNAMIC_FALLBACK
 4785    word32* rk = aes->key_C_fallback;
 4786#else
 4787    word32* rk = aes->key;
 4788#endif
 4789    word32 temp;
 4790    unsigned int i = 0;
 4791
 4792    XMEMCPY(rk, key, keySz);
 4793#if defined(LITTLE_ENDIAN_ORDER) && !defined(WOLFSSL_PIC32MZ_CRYPT) && \
 4794    (!defined(WOLFSSL_ESP32_CRYPT) || defined(NO_WOLFSSL_ESP32_CRYPT_AES)) && \
 4795    !defined(MAX3266X_AES)
 4796    /* Always reverse words when using only SW */
 4797    {
 4798        ByteReverseWords(rk, rk, keySz);
 4799    }
 4800#else
 4801    /* Sometimes reverse words when using supported HW */
 4802    #if defined(WOLFSSL_ESPIDF)
 4803        /* Some platforms may need SW fallback (e.g. AES192) */
 4804        #if defined(NEED_AES_HW_FALLBACK)
 4805        {
 4806            ESP_LOGV(TAG, "wc_AesEncrypt fallback check");
 4807            if (wc_esp32AesSupportedKeyLen(aes)) {
 4808                /* don't reverse for HW supported key lengths */
 4809            }
 4810            else {
 4811                ByteReverseWords(rk, rk, keySz);
 4812            }
 4813        }
 4814        #else
 4815            /* If we don't need SW fallback, don't need to reverse words. */
 4816        #endif /* NEED_AES_HW_FALLBACK */
 4817    #endif /* WOLFSSL_ESPIDF */
 4818#endif /* LITTLE_ENDIAN_ORDER, etc */
 4819
 4820    switch (keySz) {
 4821#if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 128 && \
 4822        defined(WOLFSSL_AES_128)
 4823    case 16:
 4824    #ifdef WOLFSSL_CHECK_MEM_ZERO
 4825        temp = (word32)-1;
 4826        wc_MemZero_Add("wc_AesSetKeyLocal temp", &temp, sizeof(temp));
 4827    #endif
 4828        while (1)
 4829        {
 4830            temp  = rk[3];
 4831            rk[4] = rk[0] ^
 4832        #ifndef WOLFSSL_AES_SMALL_TABLES
 4833                (GetTable(Te[2], GETBYTE(temp, 2)) & 0xff000000) ^
 4834                (GetTable(Te[3], GETBYTE(temp, 1)) & 0x00ff0000) ^
 4835                (GetTable(Te[0], GETBYTE(temp, 0)) & 0x0000ff00) ^
 4836                (GetTable(Te[1], GETBYTE(temp, 3)) & 0x000000ff) ^
 4837        #else
 4838                ((word32)GetTable8(Tsbox, GETBYTE(temp, 2)) << 24) ^
 4839                ((word32)GetTable8(Tsbox, GETBYTE(temp, 1)) << 16) ^
 4840                ((word32)GetTable8(Tsbox, GETBYTE(temp, 0)) <<  8) ^
 4841                ((word32)GetTable8(Tsbox, GETBYTE(temp, 3))) ^
 4842        #endif
 4843                rcon[i];
 4844            rk[5] = rk[1] ^ rk[4];
 4845            rk[6] = rk[2] ^ rk[5];
 4846            rk[7] = rk[3] ^ rk[6];
 4847            if (++i == 10)
 4848                break;
 4849            rk += 4;
 4850        }
 4851        break;
 4852#endif /* 128 */
 4853
 4854#if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 192 && \
 4855        defined(WOLFSSL_AES_192)
 4856    case 24:
 4857    #ifdef WOLFSSL_CHECK_MEM_ZERO
 4858        temp = (word32)-1;
 4859        wc_MemZero_Add("wc_AesSetKeyLocal temp", &temp, sizeof(temp));
 4860    #endif
 4861        /* for (;;) here triggers a bug in VC60 SP4 w/ Pro Pack */
 4862        while (1)
 4863        {
 4864            temp = rk[ 5];
 4865            rk[ 6] = rk[ 0] ^
 4866        #ifndef WOLFSSL_AES_SMALL_TABLES
 4867                (GetTable(Te[2], GETBYTE(temp, 2)) & 0xff000000) ^
 4868                (GetTable(Te[3], GETBYTE(temp, 1)) & 0x00ff0000) ^
 4869                (GetTable(Te[0], GETBYTE(temp, 0)) & 0x0000ff00) ^
 4870                (GetTable(Te[1], GETBYTE(temp, 3)) & 0x000000ff) ^
 4871        #else
 4872                ((word32)GetTable8(Tsbox, GETBYTE(temp, 2)) << 24) ^
 4873                ((word32)GetTable8(Tsbox, GETBYTE(temp, 1)) << 16) ^
 4874                ((word32)GetTable8(Tsbox, GETBYTE(temp, 0)) <<  8) ^
 4875                ((word32)GetTable8(Tsbox, GETBYTE(temp, 3))) ^
 4876        #endif
 4877                rcon[i];
 4878            rk[ 7] = rk[ 1] ^ rk[ 6];
 4879            rk[ 8] = rk[ 2] ^ rk[ 7];
 4880            rk[ 9] = rk[ 3] ^ rk[ 8];
 4881            if (++i == 8)
 4882                break;
 4883            rk[10] = rk[ 4] ^ rk[ 9];
 4884            rk[11] = rk[ 5] ^ rk[10];
 4885            rk += 6;
 4886        }
 4887        break;
 4888#endif /* 192 */
 4889
 4890#if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 256 && \
 4891        defined(WOLFSSL_AES_256)
 4892    case 32:
 4893    #ifdef WOLFSSL_CHECK_MEM_ZERO
 4894        temp = (word32)-1;
 4895        wc_MemZero_Add("wc_AesSetKeyLocal temp", &temp, sizeof(temp));
 4896    #endif
 4897        while (1)
 4898        {
 4899            temp = rk[ 7];
 4900            rk[ 8] = rk[ 0] ^
 4901        #ifndef WOLFSSL_AES_SMALL_TABLES
 4902                (GetTable(Te[2], GETBYTE(temp, 2)) & 0xff000000) ^
 4903                (GetTable(Te[3], GETBYTE(temp, 1)) & 0x00ff0000) ^
 4904                (GetTable(Te[0], GETBYTE(temp, 0)) & 0x0000ff00) ^
 4905                (GetTable(Te[1], GETBYTE(temp, 3)) & 0x000000ff) ^
 4906        #else
 4907                ((word32)GetTable8(Tsbox, GETBYTE(temp, 2)) << 24) ^
 4908                ((word32)GetTable8(Tsbox, GETBYTE(temp, 1)) << 16) ^
 4909                ((word32)GetTable8(Tsbox, GETBYTE(temp, 0)) <<  8) ^
 4910                ((word32)GetTable8(Tsbox, GETBYTE(temp, 3))) ^
 4911        #endif
 4912                rcon[i];
 4913            rk[ 9] = rk[ 1] ^ rk[ 8];
 4914            rk[10] = rk[ 2] ^ rk[ 9];
 4915            rk[11] = rk[ 3] ^ rk[10];
 4916            if (++i == 7)
 4917                break;
 4918            temp = rk[11];
 4919            rk[12] = rk[ 4] ^
 4920        #ifndef WOLFSSL_AES_SMALL_TABLES
 4921                (GetTable(Te[2], GETBYTE(temp, 3)) & 0xff000000) ^
 4922                (GetTable(Te[3], GETBYTE(temp, 2)) & 0x00ff0000) ^
 4923                (GetTable(Te[0], GETBYTE(temp, 1)) & 0x0000ff00) ^
 4924                (GetTable(Te[1], GETBYTE(temp, 0)) & 0x000000ff);
 4925        #else
 4926                ((word32)GetTable8(Tsbox, GETBYTE(temp, 3)) << 24) ^
 4927                ((word32)GetTable8(Tsbox, GETBYTE(temp, 2)) << 16) ^
 4928                ((word32)GetTable8(Tsbox, GETBYTE(temp, 1)) <<  8) ^
 4929                ((word32)GetTable8(Tsbox, GETBYTE(temp, 0)));
 4930        #endif
 4931            rk[13] = rk[ 5] ^ rk[12];
 4932            rk[14] = rk[ 6] ^ rk[13];
 4933            rk[15] = rk[ 7] ^ rk[14];
 4934
 4935            rk += 8;
 4936        }
 4937        break;
 4938#endif /* 256 */
 4939    } /* switch */
 4940    ForceZero(&temp, sizeof(temp));
 4941
 4942#if defined(HAVE_AES_DECRYPT) && !defined(MAX3266X_AES)
 4943    if (dir == AES_DECRYPTION) {
 4944        unsigned int j;
 4945
 4946#ifdef WC_C_DYNAMIC_FALLBACK
 4947        rk = aes->key_C_fallback;
 4948#else
 4949        rk = aes->key;
 4950#endif
 4951
 4952        /* invert the order of the round keys: */
 4953        for (i = 0, j = 4* aes->rounds; i < j; i += 4, j -= 4) {
 4954            temp = rk[i    ]; rk[i    ] = rk[j    ]; rk[j    ] = temp;
 4955            temp = rk[i + 1]; rk[i + 1] = rk[j + 1]; rk[j + 1] = temp;
 4956            temp = rk[i + 2]; rk[i + 2] = rk[j + 2]; rk[j + 2] = temp;
 4957            temp = rk[i + 3]; rk[i + 3] = rk[j + 3]; rk[j + 3] = temp;
 4958        }
 4959        ForceZero(&temp, sizeof(temp));
 4960    #if !defined(WOLFSSL_AES_SMALL_TABLES)
 4961        /* apply the inverse MixColumn transform to all round keys but the
 4962           first and the last: */
 4963        for (i = 1; i < aes->rounds; i++) {
 4964            rk += 4;
 4965            rk[0] =
 4966                GetTable(Td[0], GetTable(Te[1], GETBYTE(rk[0], 3)) & 0xff) ^
 4967                GetTable(Td[1], GetTable(Te[1], GETBYTE(rk[0], 2)) & 0xff) ^
 4968                GetTable(Td[2], GetTable(Te[1], GETBYTE(rk[0], 1)) & 0xff) ^
 4969                GetTable(Td[3], GetTable(Te[1], GETBYTE(rk[0], 0)) & 0xff);
 4970            rk[1] =
 4971                GetTable(Td[0], GetTable(Te[1], GETBYTE(rk[1], 3)) & 0xff) ^
 4972                GetTable(Td[1], GetTable(Te[1], GETBYTE(rk[1], 2)) & 0xff) ^
 4973                GetTable(Td[2], GetTable(Te[1], GETBYTE(rk[1], 1)) & 0xff) ^
 4974                GetTable(Td[3], GetTable(Te[1], GETBYTE(rk[1], 0)) & 0xff);
 4975            rk[2] =
 4976                GetTable(Td[0], GetTable(Te[1], GETBYTE(rk[2], 3)) & 0xff) ^
 4977                GetTable(Td[1], GetTable(Te[1], GETBYTE(rk[2], 2)) & 0xff) ^
 4978                GetTable(Td[2], GetTable(Te[1], GETBYTE(rk[2], 1)) & 0xff) ^
 4979                GetTable(Td[3], GetTable(Te[1], GETBYTE(rk[2], 0)) & 0xff);
 4980            rk[3] =
 4981                GetTable(Td[0], GetTable(Te[1], GETBYTE(rk[3], 3)) & 0xff) ^
 4982                GetTable(Td[1], GetTable(Te[1], GETBYTE(rk[3], 2)) & 0xff) ^
 4983                GetTable(Td[2], GetTable(Te[1], GETBYTE(rk[3], 1)) & 0xff) ^
 4984                GetTable(Td[3], GetTable(Te[1], GETBYTE(rk[3], 0)) & 0xff);
 4985        }
 4986    #endif
 4987    }
 4988#else
 4989    (void)dir;
 4990#endif /* HAVE_AES_DECRYPT */
 4991
 4992#ifdef WOLFSSL_CHECK_MEM_ZERO
 4993    wc_MemZero_Check(&temp, sizeof(temp));
 4994#else
 4995    (void)temp;
 4996#endif
 4997}
 4998#endif
 4999#else /* WC_AES_BITSLICED */
 5000/* Set the AES key and expand.
 5001 *
 5002 * @param [in]  aes    AES object.
 5003 * @param [in]  key    Block to encrypt.
 5004 * @param [in]  keySz  Number of bytes in key.
 5005 * @param [in]  dir    Direction of crypt: AES_ENCRYPTION or AES_DECRYPTION.
 5006 */
 5007static void AesSetKey_C(Aes* aes, const byte* key, word32 keySz, int dir)
 5008{
 5009    /* No need to invert when decrypting. */
 5010    (void)dir;
 5011
 5012    bs_set_key(aes->bs_key, key, keySz, aes->rounds);
 5013}
 5014#endif /* WC_AES_BITSLICED */
 5015
 5016#endif /* NEED_AES_TABLES */
 5017
 5018#ifndef WOLFSSL_RISCV_ASM
 5019    /* Software AES - SetKey */
 5020    static WARN_UNUSED_RESULT int wc_AesSetKeyLocal(
 5021        Aes* aes, const byte* userKey, word32 keylen, const byte* iv, int dir,
 5022        int checkKeyLen)
 5023    {
 5024        int ret;
 5025#if defined(WOLF_CRYPTO_CB) && defined(WOLF_CRYPTO_CB_SETKEY)
 5026        int cbRet;
 5027#endif
 5028    #ifdef WOLFSSL_IMX6_CAAM_BLOB
 5029        byte   local[32];
 5030        word32 localSz = 32;
 5031    #endif
 5032
 5033        if (aes == NULL)
 5034            return BAD_FUNC_ARG;
 5035#ifdef WC_DEBUG_CIPHER_LIFECYCLE
 5036        ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
 5037        if (ret < 0)
 5038            return ret;
 5039#endif
 5040
 5041        switch (keylen) {
 5042    #if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 128 &&     \
 5043        defined(WOLFSSL_AES_128)
 5044        case 16:
 5045    #endif
 5046    #if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 192 &&     \
 5047        defined(WOLFSSL_AES_192)
 5048        case 24:
 5049    #endif
 5050    #if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 256 &&     \
 5051        defined(WOLFSSL_AES_256)
 5052        case 32:
 5053    #endif
 5054            break;
 5055        default:
 5056            return BAD_FUNC_ARG;
 5057        }
 5058
 5059    #ifdef WOLF_CRYPTO_CB
 5060        #ifndef WOLF_CRYPTO_CB_FIND
 5061        if (aes->devId != INVALID_DEVID)
 5062        #endif
 5063        {
 5064        #ifdef WOLF_CRYPTO_CB_AES_SETKEY
 5065            ret = wc_CryptoCb_AesSetKey(aes, userKey, keylen);
 5066            if (ret == 0) {
 5067                /* Callback succeeded - SE owns the key */
 5068                aes->keylen = (int)keylen;
 5069                if (iv != NULL)
 5070                    XMEMCPY(aes->reg, iv, WC_AES_BLOCK_SIZE);
 5071                else
 5072                    XMEMSET(aes->reg, 0, WC_AES_BLOCK_SIZE);
 5073                return 0;
 5074            }
 5075            else if (ret != WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE)) {
 5076                aes->devCtx = NULL;
 5077                return ret;
 5078            }
 5079            /* CRYPTOCB_UNAVAILABLE: continue to software setup */
 5080        #endif
 5081        #ifdef WOLF_CRYPTO_CB_SETKEY
 5082            cbRet = wc_CryptoCb_SetKey(aes->devId,
 5083                WC_SETKEY_AES, aes, (void*)userKey, keylen,
 5084                (void*)iv,
 5085                (iv != NULL) ? WC_AES_BLOCK_SIZE : 0, dir);
 5086            if (cbRet != WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE))
 5087                return cbRet;
 5088            /* CRYPTOCB_UNAVAILABLE: fall through to software setup */
 5089        #endif /* WOLF_CRYPTO_CB_SETKEY */
 5090            /* Standard CryptoCB path - copy key to devKey */
 5091            if (keylen > sizeof(aes->devKey)) {
 5092                return BAD_FUNC_ARG;
 5093            }
 5094            XMEMCPY(aes->devKey, userKey, keylen);
 5095        }
 5096    #endif
 5097
 5098    #ifdef WOLFSSL_MAXQ10XX_CRYPTO
 5099        if (wc_MAXQ10XX_AesSetKey(aes, userKey, keylen) != 0) {
 5100            return WC_HW_E;
 5101        }
 5102    #endif
 5103
 5104    #ifdef WOLFSSL_IMX6_CAAM_BLOB
 5105        if (keylen == (16 + WC_CAAM_BLOB_SZ) ||
 5106            keylen == (24 + WC_CAAM_BLOB_SZ) ||
 5107            keylen == (32 + WC_CAAM_BLOB_SZ)) {
 5108            if (wc_caamOpenBlob((byte*)userKey, keylen, local, &localSz) != 0) {
 5109                return BAD_FUNC_ARG;
 5110            }
 5111
 5112            /* set local values */
 5113            userKey = local;
 5114            keylen = localSz;
 5115        }
 5116    #endif
 5117
 5118    #ifdef WOLFSSL_SECO_CAAM
 5119        /* if set to use hardware than import the key */
 5120        if (aes->devId == WOLFSSL_SECO_DEVID) {
 5121            int keyGroup = 1; /* group one was chosen arbitrarily */
 5122            unsigned int keyIdOut;
 5123            byte importiv[GCM_NONCE_MID_SZ];
 5124            int importivSz = GCM_NONCE_MID_SZ;
 5125            int keyType = 0;
 5126            WC_RNG rng;
 5127
 5128            if (wc_InitRng(&rng) != 0) {
 5129                WOLFSSL_MSG("RNG init for IV failed");
 5130                return WC_HW_E;
 5131            }
 5132
 5133            if (wc_RNG_GenerateBlock(&rng, importiv, importivSz) != 0) {
 5134                WOLFSSL_MSG("Generate IV failed");
 5135                wc_FreeRng(&rng);
 5136                return WC_HW_E;
 5137            }
 5138            wc_FreeRng(&rng);
 5139
 5140            if (iv)
 5141                XMEMCPY(aes->reg, iv, WC_AES_BLOCK_SIZE);
 5142            else
 5143                XMEMSET(aes->reg, 0, WC_AES_BLOCK_SIZE);
 5144
 5145            switch (keylen) {
 5146                case AES_128_KEY_SIZE: keyType = CAAM_KEYTYPE_AES128; break;
 5147                case AES_192_KEY_SIZE: keyType = CAAM_KEYTYPE_AES192; break;
 5148                case AES_256_KEY_SIZE: keyType = CAAM_KEYTYPE_AES256; break;
 5149            }
 5150
 5151            keyIdOut = wc_SECO_WrapKey(0, (byte*)userKey, keylen, importiv,
 5152                importivSz, keyType, CAAM_KEY_TRANSIENT, keyGroup);
 5153            if (keyIdOut == 0) {
 5154                return WC_HW_E;
 5155            }
 5156            aes->blackKey = keyIdOut;
 5157            return 0;
 5158        }
 5159    #endif
 5160
 5161    #if defined(WOLF_CRYPTO_CB) || (defined(WOLFSSL_DEVCRYPTO) && \
 5162        (defined(WOLFSSL_DEVCRYPTO_AES) || defined(WOLFSSL_DEVCRYPTO_CBC))) || \
 5163        (defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)) || \
 5164        defined(WOLFSSL_NXP_HASHCRYPT_AES)
 5165        #ifdef WOLF_CRYPTO_CB
 5166        #ifndef WOLF_CRYPTO_CB_FIND
 5167        if (aes->devId != INVALID_DEVID)
 5168        #endif
 5169        #endif
 5170        {
 5171            if (keylen > sizeof(aes->devKey)) {
 5172                return BAD_FUNC_ARG;
 5173            }
 5174            XMEMCPY(aes->devKey, userKey, keylen);
 5175        }
 5176    #endif
 5177
 5178    #ifdef WOLF_CRYPTO_CB_ONLY_AES
 5179        /* No software AES schedule under CB_ONLY: aes->key[] (round keys) are
 5180         * unused because the static wc_AesEncrypt/wc_AesDecrypt are cryptocb-
 5181         * ECB shims. aes->rounds is still populated because wc_AesGetKeySize()
 5182         * reads it as the source of truth for the configured key size. */
 5183        aes->keylen = (int)keylen;
 5184        aes->rounds = (keylen / 4) + 6;
 5185        #if defined(WOLFSSL_AES_COUNTER) || defined(WOLFSSL_AES_CFB) || \
 5186            defined(WOLFSSL_AES_OFB) || defined(WOLFSSL_AES_XTS) || \
 5187            defined(WOLFSSL_AES_CTS)
 5188        aes->left = 0;
 5189        #endif
 5190        (void)dir;
 5191        return wc_AesSetIV(aes, iv);
 5192    #endif
 5193
 5194    #if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE < 256
 5195        if (checkKeyLen) {
 5196            /* Check key length only when AES_MAX_KEY_SIZE doesn't allow
 5197             * all key sizes. Otherwise this condition is never true. */
 5198            if (keylen > (AES_MAX_KEY_SIZE / 8)) {
 5199                return BAD_FUNC_ARG;
 5200            }
 5201        }
 5202    #else
 5203        (void) checkKeyLen;
 5204    #endif
 5205
 5206    #if defined(WOLFSSL_AES_COUNTER) || defined(WOLFSSL_AES_CFB) || \
 5207        defined(WOLFSSL_AES_OFB) || defined(WOLFSSL_AES_XTS) || \
 5208        defined(WOLFSSL_AES_CTS)
 5209        aes->left = 0;
 5210    #endif
 5211
 5212        aes->keylen = (int)keylen;
 5213        aes->rounds = (keylen/4) + 6;
 5214        ret = wc_AesSetIV(aes, iv);
 5215        if (ret != 0)
 5216            return ret;
 5217
 5218#ifdef WC_C_DYNAMIC_FALLBACK
 5219#ifdef NEED_AES_TABLES
 5220        AesSetKey_C(aes, userKey, keylen, dir);
 5221#endif /* NEED_AES_TABLES */
 5222#endif /* WC_C_DYNAMIC_FALLBACK */
 5223
 5224    #ifdef WOLFSSL_AESNI
 5225
 5226       /* The dynamics for determining whether AES-NI will be used are tricky.
 5227        *
 5228        * First, we check for CPU support and cache the result -- if AES-NI is
 5229        * missing, we always shortcut to the AesSetKey_C() path.
 5230        *
 5231        * Second, if the CPU supports AES-NI, we confirm on a per-call basis
 5232        * that it's safe to use in the caller context, using
 5233        * SAVE_VECTOR_REGISTERS2().  This is an always-true no-op in user-space
 5234        * builds, but has substantive logic behind it in kernel module builds.
 5235        *
 5236        * The outcome when SAVE_VECTOR_REGISTERS2() fails depends on
 5237        * WC_C_DYNAMIC_FALLBACK -- if that's defined, we return immediately with
 5238        * success but with AES-NI disabled (the earlier AesSetKey_C() allows
 5239        * future encrypt/decrypt calls to succeed), otherwise we fail.
 5240        *
 5241        * Upon successful return, aes->use_aesni will have a zero value if
 5242        * AES-NI is disabled, and a nonzero value if it's enabled.
 5243        *
 5244        * An additional, optional semantic is available via
 5245        * WC_FLAG_DONT_USE_VECTOR_OPS, and is used in some kernel module builds
 5246        * to let the caller inhibit AES-NI.  When this macro is defined,
 5247        * wc_AesInit() before wc_AesSetKey() is imperative, to avoid a read of
 5248        * uninitialized data in aes->use_aesni.  That's why support for
 5249        * WC_FLAG_DONT_USE_VECTOR_OPS must remain optional -- wc_AesInit() was
 5250        * only added in release 3.11.0, so legacy applications inevitably call
 5251        * wc_AesSetKey() on uninitialized Aes contexts.  This must continue to
 5252        * function correctly with default build settings.
 5253        */
 5254
 5255        if (checkedAESNI == 0) {
 5256            haveAESNI  = Check_CPU_support_AES();
 5257            checkedAESNI = 1;
 5258        }
 5259        if (haveAESNI
 5260#if defined(WC_FLAG_DONT_USE_VECTOR_OPS) && !defined(WC_C_DYNAMIC_FALLBACK)
 5261            && (aes->use_aesni != WC_FLAG_DONT_USE_VECTOR_OPS)
 5262#endif
 5263            )
 5264        {
 5265#if defined(WC_FLAG_DONT_USE_VECTOR_OPS)
 5266            if (aes->use_aesni == WC_FLAG_DONT_USE_VECTOR_OPS) {
 5267                aes->use_aesni = 0;
 5268                return 0;
 5269            }
 5270#endif
 5271            aes->use_aesni = 0;
 5272            #ifdef WOLFSSL_KERNEL_MODE
 5273            /* runtime alignment check */
 5274            if ((wc_ptr_t)&aes->key & (wc_ptr_t)0xf) {
 5275                ret = BAD_ALIGN_E;
 5276            }
 5277            else
 5278            #endif /* WOLFSSL_KERNEL_MODE */
 5279            {
 5280                ret = SAVE_VECTOR_REGISTERS2();
 5281            }
 5282            if (ret == 0) {
 5283                if (dir == AES_ENCRYPTION)
 5284                    ret = AES_set_encrypt_key_AESNI(userKey, (int)keylen * 8, aes);
 5285#ifdef HAVE_AES_DECRYPT
 5286                else
 5287                    ret = AES_set_decrypt_key_AESNI(userKey, (int)keylen * 8, aes);
 5288#endif
 5289
 5290                RESTORE_VECTOR_REGISTERS();
 5291
 5292                if (ret == 0)
 5293                    aes->use_aesni = 1;
 5294                else {
 5295#ifdef WC_C_DYNAMIC_FALLBACK
 5296                    ret = 0;
 5297#endif
 5298                }
 5299                return ret;
 5300            } else {
 5301#ifdef WC_C_DYNAMIC_FALLBACK
 5302                return 0;
 5303#else
 5304                return ret;
 5305#endif
 5306            }
 5307        }
 5308        else {
 5309            aes->use_aesni = 0;
 5310#ifdef WC_C_DYNAMIC_FALLBACK
 5311            /* If WC_C_DYNAMIC_FALLBACK, we already called AesSetKey_C()
 5312             * above.
 5313             */
 5314            return 0;
 5315#endif
 5316        }
 5317    #endif /* WOLFSSL_AESNI */
 5318
 5319#ifndef WC_C_DYNAMIC_FALLBACK
 5320
 5321#if defined(WOLFSSL_ARMASM)
 5322#if !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
 5323    #ifndef __aarch64__
 5324        AES_set_key_AARCH32(userKey, keylen, (byte*)aes->key, dir);
 5325    #else
 5326        Check_CPU_support_HwCrypto(aes);
 5327        if (aes->use_aes_hw_crypto) {
 5328            AES_set_key_AARCH64(userKey, keylen, (byte*)aes->key, dir);
 5329        }
 5330        else
 5331    #endif /* __aarch64__ */
 5332#endif /* !WOLFSSL_ARMASM_NO_HW_CRYPTO */
 5333    #if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON) && \
 5334        defined(WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP)
 5335        {
 5336            AES_set_encrypt_key_NEON(userKey, keylen * 8, (byte*)aes->key);
 5337        #ifdef HAVE_AES_DECRYPT
 5338            if (dir == AES_DECRYPTION) {
 5339                AES_invert_key_NEON((byte*)aes->key, aes->rounds);
 5340            }
 5341        #else
 5342            (void)dir;
 5343        #endif
 5344        }
 5345    #elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
 5346        {
 5347            AES_set_encrypt_key(userKey, keylen * 8, (byte*)aes->key);
 5348        #ifdef HAVE_AES_DECRYPT
 5349            if (dir == AES_DECRYPTION) {
 5350                AES_invert_key((byte*)aes->key, aes->rounds);
 5351            }
 5352        #else
 5353            (void)dir;
 5354        #endif
 5355        }
 5356    #endif
 5357        return 0;
 5358#else
 5359
 5360    #ifdef WOLFSSL_KCAPI_AES
 5361        XMEMCPY(aes->devKey, userKey, keylen);
 5362        if (aes->init != 0) {
 5363            kcapi_cipher_destroy(aes->handle);
 5364            aes->handle = NULL;
 5365            aes->init = 0;
 5366        }
 5367        (void)dir;
 5368    #endif
 5369
 5370        if (keylen > sizeof(aes->key)) {
 5371            return BAD_FUNC_ARG;
 5372        }
 5373#if defined(WOLFSSL_HAVE_PSA) && !defined(WOLFSSL_PSA_NO_AES)
 5374        return wc_psa_aes_set_key(aes, userKey, keylen, (uint8_t*)iv,
 5375                                  ((psa_algorithm_t)0), dir);
 5376#endif
 5377
 5378#if defined(WOLFSSL_SE050) && defined(WOLFSSL_SE050_CRYPT)
 5379        /* wolfSSL HostCrypto in SE05x SDK can request to use SW crypto
 5380         * instead of SE05x crypto by setting useSWCrypt */
 5381        if (aes->useSWCrypt == 0) {
 5382            ret = se050_aes_set_key(aes, userKey, keylen, iv, dir);
 5383            if (ret == 0) {
 5384                ret = wc_AesSetIV(aes, iv);
 5385            }
 5386            return ret;
 5387        }
 5388#endif
 5389#if defined(WOLFSSL_MICROCHIP_TA100) && defined(WOLFSSL_MICROCHIP_AESGCM)
 5390        if (keylen == TA_KEY_TYPE_AES128_SIZE) {
 5391            ret = wc_Microchip_aes_set_key(aes, userKey, keylen, iv, dir);
 5392            if (ret != 0) {
 5393                return ret;
 5394            }
 5395            ret = wc_AesSetIV(aes, iv);
 5396            if (ret != 0) {
 5397                return ret;
 5398            }
 5399        }
 5400#endif
 5401        XMEMCPY(aes->key, userKey, keylen);
 5402
 5403#ifndef WC_AES_BITSLICED
 5404    #if defined(LITTLE_ENDIAN_ORDER) && !defined(WOLFSSL_PIC32MZ_CRYPT) && \
 5405        (!defined(WOLFSSL_ESP32_CRYPT) || defined(NO_WOLFSSL_ESP32_CRYPT_AES)) \
 5406        && !defined(MAX3266X_AES)
 5407
 5408        /* software */
 5409        ByteReverseWords(aes->key, aes->key, keylen);
 5410
 5411    #elif defined(WOLFSSL_ESP32_CRYPT) && !defined(NO_WOLFSSL_ESP32_CRYPT_AES)
 5412        if (wc_esp32AesSupportedKeyLen(aes)) {
 5413            /* supported lengths don't get reversed */
 5414            ESP_LOGV(TAG, "wc_AesSetKeyLocal (no ByteReverseWords)");
 5415        }
 5416        else {
 5417            word32* rk = aes->key;
 5418
 5419            /* For example, the ESP32-S3 does not support HW for len = 24,
 5420             * so fall back to SW */
 5421        #ifdef DEBUG_WOLFSSL
 5422            ESP_LOGW(TAG, "wc_AesSetKeyLocal ByteReverseWords");
 5423        #endif
 5424            XMEMCPY(rk, userKey, keylen);
 5425            /* When not ESP32 HW, we need to reverse endianness */
 5426            ByteReverseWords(rk, rk, keylen);
 5427        }
 5428    #endif
 5429
 5430    #ifdef WOLFSSL_IMXRT_DCP
 5431        {
 5432            /* Implemented in wolfcrypt/src/port/nxp/dcp_port.c */
 5433            word32 temp = 0;
 5434            if (keylen == 16)
 5435                temp = DCPAesSetKey(aes, userKey, keylen, iv, dir);
 5436            if (temp != 0)
 5437                return WC_HW_E;
 5438        }
 5439    #endif
 5440#endif /* !WC_AES_BITSLICED */
 5441
 5442#ifdef NEED_AES_TABLES
 5443        AesSetKey_C(aes, userKey, keylen, dir);
 5444#endif /* NEED_AES_TABLES */
 5445
 5446#if defined(WOLFSSL_SCE) && !defined(WOLFSSL_SCE_NO_AES)
 5447        XMEMCPY((byte*)aes->key, userKey, keylen);
 5448        if (WOLFSSL_SCE_GSCE_HANDLE.p_cfg->endian_flag == CRYPTO_WORD_ENDIAN_BIG) {
 5449            ByteReverseWords(aes->key, aes->key, 32);
 5450        }
 5451#endif
 5452
 5453    #if defined(WOLFSSL_DEVCRYPTO) && \
 5454        (defined(WOLFSSL_DEVCRYPTO_AES) || defined(WOLFSSL_DEVCRYPTO_CBC))
 5455        aes->ctx.cfd = -1;
 5456    #endif
 5457    #ifdef WOLFSSL_IMX6_CAAM_BLOB
 5458        ForceZero(local, sizeof(local));
 5459    #endif
 5460        return ret;
 5461#endif
 5462
 5463#endif /* !WC_C_DYNAMIC_FALLBACK */
 5464
 5465    } /* wc_AesSetKeyLocal */
 5466
 5467    int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen,
 5468            const byte* iv, int dir)
 5469    {
 5470        if (aes == NULL) {
 5471            return BAD_FUNC_ARG;
 5472        }
 5473        if (keylen > sizeof(aes->key)) {
 5474            return BAD_FUNC_ARG;
 5475        }
 5476
 5477    /* sometimes hardware may not support all keylengths (e.g. ESP32-S3) */
 5478    #if defined(WOLFSSL_ESPIDF) && defined(NEED_AES_HW_FALLBACK)
 5479        ESP_LOGV(TAG, "wc_AesSetKey fallback check %d", keylen);
 5480        if (wc_esp32AesSupportedKeyLenValue(keylen)) {
 5481            ESP_LOGV(TAG, "wc_AesSetKey calling wc_AesSetKey_for_ESP32");
 5482            return wc_AesSetKey_for_ESP32(aes, userKey, keylen, iv, dir);
 5483        }
 5484        else {
 5485        #if  defined(WOLFSSL_HW_METRICS)
 5486            /* It is interesting to know how many times we could not complete
 5487             * AES in hardware due to unsupported lengths. */
 5488            wc_esp32AesUnupportedLengthCountAdd();
 5489        #endif
 5490        #ifdef DEBUG_WOLFSSL
 5491            ESP_LOGW(TAG, "wc_AesSetKey HW Fallback, unsupported keylen = %d",
 5492                           keylen);
 5493        #endif
 5494        }
 5495    #endif /* WOLFSSL_ESPIDF && NEED_AES_HW_FALLBACK */
 5496
 5497        return wc_AesSetKeyLocal(aes, userKey, keylen, iv, dir, 1);
 5498
 5499    } /* wc_AesSetKey() */
 5500#endif
 5501
 5502    #if defined(WOLFSSL_AES_DIRECT) || defined(WOLFSSL_AES_COUNTER)
 5503        /* AES-CTR and AES-DIRECT need to use this for key setup */
 5504        /* This function allows key sizes that are not 128/192/256 bits */
 5505    int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
 5506                           const byte* iv, int dir)
 5507    {
 5508        if (aes == NULL) {
 5509            return BAD_FUNC_ARG;
 5510        }
 5511        if (keylen > sizeof(aes->key)) {
 5512            return BAD_FUNC_ARG;
 5513        }
 5514
 5515        return wc_AesSetKeyLocal(aes, userKey, keylen, iv, dir, 0);
 5516    }
 5517    #endif /* WOLFSSL_AES_DIRECT || WOLFSSL_AES_COUNTER */
 5518#endif /* wc_AesSetKey block */
 5519
 5520
 5521/* wc_AesSetIV is shared between software and hardware */
 5522int wc_AesSetIV(Aes* aes, const byte* iv)
 5523{
 5524    if (aes == NULL)
 5525        return BAD_FUNC_ARG;
 5526
 5527#ifdef WC_DEBUG_CIPHER_LIFECYCLE
 5528    {
 5529        int ret = wc_debug_CipherLifecycleCheck(aes->CipherLifecycleTag, 0);
 5530        if (ret < 0)
 5531            return ret;
 5532    }
 5533#endif
 5534
 5535    if (iv)
 5536        XMEMCPY(aes->reg, iv, WC_AES_BLOCK_SIZE);
 5537    else
 5538        XMEMSET(aes->reg,  0, WC_AES_BLOCK_SIZE);
 5539
 5540#if defined(WOLFSSL_AES_COUNTER) || defined(WOLFSSL_AES_CFB) || \
 5541    defined(WOLFSSL_AES_OFB) || defined(WOLFSSL_AES_XTS) || \
 5542    defined(WOLFSSL_AES_CTS)
 5543    /* Clear any unused bytes from last cipher op. */
 5544    aes->left = 0;
 5545#endif
 5546
 5547    return 0;
 5548}
 5549
 5550#ifdef WOLFSSL_AESNI
 5551
 5552#ifdef WC_C_DYNAMIC_FALLBACK
 5553
 5554#define VECTOR_REGISTERS_PUSH {                                      \
 5555        int orig_use_aesni = aes->use_aesni;                         \
 5556        if (aes->use_aesni && (SAVE_VECTOR_REGISTERS2() != 0)) {     \
 5557            aes->use_aesni = 0;                                      \
 5558        }                                                            \
 5559        WC_DO_NOTHING
 5560
 5561#define VECTOR_REGISTERS_POP                                         \
 5562        if (aes->use_aesni)                                          \
 5563            RESTORE_VECTOR_REGISTERS();                              \
 5564        else                                                         \
 5565            aes->use_aesni = orig_use_aesni;                         \
 5566    }                                                                \
 5567    WC_DO_NOTHING
 5568
 5569#elif defined(SAVE_VECTOR_REGISTERS2_DOES_NOTHING)
 5570
 5571#define VECTOR_REGISTERS_PUSH { \
 5572        WC_DO_NOTHING
 5573
 5574#define VECTOR_REGISTERS_POP                                         \
 5575    }                                                                \
 5576    WC_DO_NOTHING
 5577
 5578#else
 5579
 5580#define VECTOR_REGISTERS_PUSH { \
 5581        if (aes->use_aesni && ((ret = SAVE_VECTOR_REGISTERS2()) != 0)) { \
 5582            return ret;                                                  \
 5583        }                                                                \
 5584        WC_DO_NOTHING
 5585
 5586#define VECTOR_REGISTERS_POP \
 5587        if (aes->use_aesni) {                                            \
 5588            RESTORE_VECTOR_REGISTERS();                                  \
 5589        }                                                                \
 5590    }                                                                    \
 5591    WC_DO_NOTHING
 5592
 5593#endif
 5594
 5595#else /* !WOLFSSL_AESNI */
 5596
 5597#define VECTOR_REGISTERS_PUSH WC_DO_NOTHING
 5598#define VECTOR_REGISTERS_POP WC_DO_NOTHING
 5599
 5600#endif /* !WOLFSSL_AESNI */
 5601
 5602
 5603/* AES-DIRECT */
 5604#if defined(WOLFSSL_AES_DIRECT)
 5605    #if defined(HAVE_COLDFIRE_SEC)
 5606        #error "Coldfire SEC doesn't yet support AES direct"
 5607
 5608    #elif defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES) && \
 5609        !defined(WOLFSSL_QNX_CAAM)
 5610        /* implemented in wolfcrypt/src/port/caam/caam_aes.c */
 5611
 5612    #elif defined(WOLFSSL_AFALG)
 5613        /* implemented in wolfcrypt/src/port/af_alg/afalg_aes.c */
 5614
 5615    #elif defined(WOLFSSL_DEVCRYPTO_AES)
 5616        /* implemented in wolfcrypt/src/port/devcrypt/devcrypto_aes.c */
 5617
 5618    #else
 5619
 5620        /* Allow direct access to one block encrypt */
 5621        /* Note, the in and out args are swapped compared to wc_AesEncrypt(). */
 5622        int wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in)
 5623        {
 5624            int ret;
 5625
 5626            if (aes == NULL || out == NULL || in == NULL)
 5627                return BAD_FUNC_ARG;
 5628            VECTOR_REGISTERS_PUSH;
 5629            ret = wc_AesEncrypt(aes, in, out);
 5630            VECTOR_REGISTERS_POP;
 5631            return ret;
 5632        }
 5633
 5634        /* vector reg save/restore is explicit in all below calls to
 5635         * wc_Aes{En,De}cryptDirect(), so bypass the public version with a
 5636         * macro.
 5637         */
 5638        #define wc_AesEncryptDirect(aes, out, in) wc_AesEncrypt(aes, in, out)
 5639
 5640        #ifdef HAVE_AES_DECRYPT
 5641        /* Allow direct access to one block decrypt */
 5642        /* Note, the in and out args are swapped compared to wc_AesDecrypt(). */
 5643        int wc_AesDecryptDirect(Aes* aes, byte* out, const byte* in)
 5644        {
 5645            int ret;
 5646
 5647            if (aes == NULL)
 5648                return BAD_FUNC_ARG;
 5649            VECTOR_REGISTERS_PUSH;
 5650            ret = wc_AesDecrypt(aes, in, out);
 5651            VECTOR_REGISTERS_POP;
 5652            return ret;
 5653        }
 5654
 5655        #define wc_AesDecryptDirect(aes, out, in) wc_AesDecrypt(aes, in, out)
 5656
 5657        #endif /* HAVE_AES_DECRYPT */
 5658    #endif /* AES direct block */
 5659#endif /* WOLFSSL_AES_DIRECT */
 5660
 5661
 5662/* AES-CBC */
 5663#ifdef HAVE_AES_CBC
 5664#if defined(STM32_CRYPTO)
 5665
 5666#ifdef WOLFSSL_STM32U5_DHUK
 5667    int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 5668    {
 5669        int ret = 0;
 5670        CRYP_HandleTypeDef hcryp;
 5671        word32 blocks = (sz / WC_AES_BLOCK_SIZE);
 5672
 5673#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 5674        if (sz % WC_AES_BLOCK_SIZE) {
 5675            return BAD_LENGTH_E;
 5676        }
 5677#endif
 5678        if (blocks == 0)
 5679            return 0;
 5680
 5681        ret = wolfSSL_CryptHwMutexLock();
 5682        if (ret != 0) {
 5683            return ret;
 5684        }
 5685
 5686        if (aes->devId == WOLFSSL_STM32U5_DHUK_WRAPPED_DEVID) {
 5687            CRYP_ConfigTypeDef Config;
 5688
 5689            XMEMSET(&Config, 0, sizeof(Config));
 5690            ret = wc_Stm32_Aes_UnWrap(aes, &hcryp, (const byte*)aes->key, aes->keylen,
 5691                (const byte*)aes->dhukIV, aes->dhukIVLen);
 5692
 5693            /* reconfigure for using unwrapped key now */
 5694            HAL_CRYP_GetConfig(&hcryp, &Config);
 5695            Config.KeyMode   = CRYP_KEYMODE_NORMAL;
 5696            Config.KeySelect = CRYP_KEYSEL_NORMAL;
 5697            Config.Algorithm = CRYP_AES_CBC;
 5698            ByteReverseWords(aes->reg, aes->reg, WC_AES_BLOCK_SIZE);
 5699            Config.pInitVect = (STM_CRYPT_TYPE*)aes->reg;
 5700            HAL_CRYP_SetConfig(&hcryp, &Config);
 5701        }
 5702        else {
 5703            ret = wc_Stm32_Aes_Init(aes, &hcryp, 1);
 5704            if (ret != 0) {
 5705                wolfSSL_CryptHwMutexUnLock();
 5706                return ret;
 5707            }
 5708            hcryp.Init.Algorithm  = CRYP_AES_CBC;
 5709            ByteReverseWords(aes->reg, aes->reg, WC_AES_BLOCK_SIZE);
 5710            hcryp.Init.pInitVect = (STM_CRYPT_TYPE*)aes->reg;
 5711            ret = HAL_CRYP_Init(&hcryp);
 5712        }
 5713
 5714        if (ret == HAL_OK) {
 5715            ret = HAL_CRYP_Encrypt(&hcryp, (uint32_t*)in, blocks * WC_AES_BLOCK_SIZE,
 5716                (uint32_t*)out, STM32_HAL_TIMEOUT);
 5717            if (ret != HAL_OK) {
 5718                ret = WC_TIMEOUT_E;
 5719            }
 5720
 5721            /* store iv for next call */
 5722            XMEMCPY(aes->reg, out + sz - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
 5723        }
 5724
 5725        HAL_CRYP_DeInit(&hcryp);
 5726
 5727        wolfSSL_CryptHwMutexUnLock();
 5728        wc_Stm32_Aes_Cleanup();
 5729
 5730        return ret;
 5731    }
 5732    #ifdef HAVE_AES_DECRYPT
 5733    int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 5734    {
 5735        int ret = 0;
 5736        CRYP_HandleTypeDef hcryp;
 5737        word32 blocks = (sz / WC_AES_BLOCK_SIZE);
 5738
 5739#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 5740        if (sz % WC_AES_BLOCK_SIZE) {
 5741            return BAD_LENGTH_E;
 5742        }
 5743#endif
 5744        if (blocks == 0)
 5745            return 0;
 5746
 5747        ret = wolfSSL_CryptHwMutexLock();
 5748        if (ret != 0) {
 5749            return ret;
 5750        }
 5751
 5752        if (aes->devId == WOLFSSL_STM32U5_DHUK_WRAPPED_DEVID) {
 5753            CRYP_ConfigTypeDef Config;
 5754
 5755            XMEMSET(&Config, 0, sizeof(Config));
 5756            ret = wc_Stm32_Aes_UnWrap(aes, &hcryp, (const byte*)aes->key, aes->keylen,
 5757                aes->dhukIV, aes->dhukIVLen);
 5758
 5759            /* reconfigure for using unwrapped key now */
 5760            HAL_CRYP_GetConfig(&hcryp, &Config);
 5761            Config.KeyMode   = CRYP_KEYMODE_NORMAL;
 5762            Config.KeySelect = CRYP_KEYSEL_NORMAL;
 5763            Config.Algorithm = CRYP_AES_CBC;
 5764            ByteReverseWords(aes->reg, aes->reg, WC_AES_BLOCK_SIZE);
 5765            Config.pInitVect = (STM_CRYPT_TYPE*)aes->reg;
 5766            HAL_CRYP_SetConfig(&hcryp, &Config);
 5767        }
 5768        else {
 5769            ret = wc_Stm32_Aes_Init(aes, &hcryp, 1);
 5770            if (ret != 0) {
 5771                wolfSSL_CryptHwMutexUnLock();
 5772                return ret;
 5773            }
 5774            hcryp.Init.Algorithm  = CRYP_AES_CBC;
 5775            ByteReverseWords(aes->reg, aes->reg, WC_AES_BLOCK_SIZE);
 5776            hcryp.Init.pInitVect = (STM_CRYPT_TYPE*)aes->reg;
 5777            ret = HAL_CRYP_Init(&hcryp);
 5778        }
 5779
 5780        if (ret == HAL_OK) {
 5781            /* if input and output same will overwrite input iv */
 5782            XMEMCPY(aes->tmp, in + sz - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
 5783            ret = HAL_CRYP_Decrypt(&hcryp, (uint32_t*)in, blocks * WC_AES_BLOCK_SIZE,
 5784                (uint32_t*)out, STM32_HAL_TIMEOUT);
 5785            if (ret != HAL_OK) {
 5786                ret = WC_TIMEOUT_E;
 5787            }
 5788
 5789            /* store iv for next call */
 5790            XMEMCPY(aes->reg, aes->tmp, WC_AES_BLOCK_SIZE);
 5791        }
 5792
 5793        HAL_CRYP_DeInit(&hcryp);
 5794        wolfSSL_CryptHwMutexUnLock();
 5795        wc_Stm32_Aes_Cleanup();
 5796
 5797        return ret;
 5798    }
 5799    #endif /* HAVE_AES_DECRYPT */
 5800
 5801#elif defined(WOLFSSL_STM32_CUBEMX)
 5802    int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 5803    {
 5804        int ret = 0;
 5805        CRYP_HandleTypeDef hcryp;
 5806        word32 blocks = (sz / WC_AES_BLOCK_SIZE);
 5807
 5808#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 5809        if (sz % WC_AES_BLOCK_SIZE) {
 5810            return BAD_LENGTH_E;
 5811        }
 5812#endif
 5813        if (blocks == 0)
 5814            return 0;
 5815
 5816        ret = wc_Stm32_Aes_Init(aes, &hcryp, 0);
 5817        if (ret != 0)
 5818            return ret;
 5819
 5820        ret = wolfSSL_CryptHwMutexLock();
 5821        if (ret != 0) {
 5822            return ret;
 5823        }
 5824
 5825    #if defined(STM32_HAL_V2)
 5826        hcryp.Init.Algorithm  = CRYP_AES_CBC;
 5827        ByteReverseWords(aes->reg, aes->reg, WC_AES_BLOCK_SIZE);
 5828    #elif defined(STM32_CRYPTO_AES_ONLY)
 5829        hcryp.Init.OperatingMode = CRYP_ALGOMODE_ENCRYPT;
 5830        hcryp.Init.ChainingMode  = CRYP_CHAINMODE_AES_CBC;
 5831        hcryp.Init.KeyWriteFlag  = CRYP_KEY_WRITE_ENABLE;
 5832    #endif
 5833        hcryp.Init.pInitVect = (STM_CRYPT_TYPE*)aes->reg;
 5834        ret = HAL_CRYP_Init(&hcryp);
 5835
 5836        if (ret == HAL_OK) {
 5837        #if defined(STM32_HAL_V2)
 5838            ret = HAL_CRYP_Encrypt(&hcryp, (uint32_t*)in, blocks * WC_AES_BLOCK_SIZE,
 5839                (uint32_t*)out, STM32_HAL_TIMEOUT);
 5840        #elif defined(STM32_CRYPTO_AES_ONLY)
 5841            ret = HAL_CRYPEx_AES(&hcryp, (uint8_t*)in, blocks * WC_AES_BLOCK_SIZE,
 5842                out, STM32_HAL_TIMEOUT);
 5843        #else
 5844            ret = HAL_CRYP_AESCBC_Encrypt(&hcryp, (uint8_t*)in,
 5845                                        blocks * WC_AES_BLOCK_SIZE,
 5846                                        out, STM32_HAL_TIMEOUT);
 5847        #endif
 5848        }
 5849        if (ret != HAL_OK) {
 5850            ret = WC_TIMEOUT_E;
 5851        }
 5852
 5853        /* store iv for next call */
 5854        XMEMCPY(aes->reg, out + sz - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
 5855
 5856        HAL_CRYP_DeInit(&hcryp);
 5857
 5858        wolfSSL_CryptHwMutexUnLock();
 5859        wc_Stm32_Aes_Cleanup();
 5860
 5861        return ret;
 5862    }
 5863    #ifdef HAVE_AES_DECRYPT
 5864    int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 5865    {
 5866        int ret = 0;
 5867        CRYP_HandleTypeDef hcryp;
 5868        word32 blocks = (sz / WC_AES_BLOCK_SIZE);
 5869
 5870#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 5871        if (sz % WC_AES_BLOCK_SIZE) {
 5872            return BAD_LENGTH_E;
 5873        }
 5874#endif
 5875        if (blocks == 0)
 5876            return 0;
 5877
 5878        ret = wc_Stm32_Aes_Init(aes, &hcryp, 0);
 5879        if (ret != 0)
 5880            return ret;
 5881
 5882        ret = wolfSSL_CryptHwMutexLock();
 5883        if (ret != 0) {
 5884            return ret;
 5885        }
 5886
 5887        /* if input and output same will overwrite input iv */
 5888        XMEMCPY(aes->tmp, in + sz - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
 5889
 5890    #if defined(STM32_HAL_V2)
 5891        hcryp.Init.Algorithm  = CRYP_AES_CBC;
 5892        ByteReverseWords(aes->reg, aes->reg, WC_AES_BLOCK_SIZE);
 5893    #elif defined(STM32_CRYPTO_AES_ONLY)
 5894        hcryp.Init.OperatingMode = CRYP_ALGOMODE_KEYDERIVATION_DECRYPT;
 5895        hcryp.Init.ChainingMode  = CRYP_CHAINMODE_AES_CBC;
 5896        hcryp.Init.KeyWriteFlag  = CRYP_KEY_WRITE_ENABLE;
 5897    #endif
 5898
 5899        hcryp.Init.pInitVect = (STM_CRYPT_TYPE*)aes->reg;
 5900        ret = HAL_CRYP_Init(&hcryp);
 5901
 5902        if (ret == HAL_OK) {
 5903        #if defined(STM32_HAL_V2)
 5904            ret = HAL_CRYP_Decrypt(&hcryp, (uint32_t*)in, blocks * WC_AES_BLOCK_SIZE,
 5905                (uint32_t*)out, STM32_HAL_TIMEOUT);
 5906        #elif defined(STM32_CRYPTO_AES_ONLY)
 5907            ret = HAL_CRYPEx_AES(&hcryp, (uint8_t*)in, blocks * WC_AES_BLOCK_SIZE,
 5908                out, STM32_HAL_TIMEOUT);
 5909        #else
 5910            ret = HAL_CRYP_AESCBC_Decrypt(&hcryp, (uint8_t*)in,
 5911                                        blocks * WC_AES_BLOCK_SIZE,
 5912                out, STM32_HAL_TIMEOUT);
 5913        #endif
 5914        }
 5915        if (ret != HAL_OK) {
 5916            ret = WC_TIMEOUT_E;
 5917        }
 5918
 5919        /* store iv for next call */
 5920        XMEMCPY(aes->reg, aes->tmp, WC_AES_BLOCK_SIZE);
 5921
 5922        HAL_CRYP_DeInit(&hcryp);
 5923        wolfSSL_CryptHwMutexUnLock();
 5924        wc_Stm32_Aes_Cleanup();
 5925
 5926        return ret;
 5927    }
 5928    #endif /* HAVE_AES_DECRYPT */
 5929
 5930#else /* Standard Peripheral Library */
 5931    int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 5932    {
 5933        int ret;
 5934        word32 *iv;
 5935        CRYP_InitTypeDef cryptInit;
 5936        CRYP_KeyInitTypeDef keyInit;
 5937        CRYP_IVInitTypeDef ivInit;
 5938        word32 blocks = (sz / WC_AES_BLOCK_SIZE);
 5939
 5940#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 5941        if (sz % WC_AES_BLOCK_SIZE) {
 5942            return BAD_LENGTH_E;
 5943        }
 5944#endif
 5945        if (blocks == 0)
 5946            return 0;
 5947
 5948        ret = wc_Stm32_Aes_Init(aes, &cryptInit, &keyInit);
 5949        if (ret != 0)
 5950            return ret;
 5951
 5952        ret = wolfSSL_CryptHwMutexLock();
 5953        if (ret != 0) {
 5954            return ret;
 5955        }
 5956
 5957        /* reset registers to their default values */
 5958        CRYP_DeInit();
 5959
 5960        /* set key */
 5961        CRYP_KeyInit(&keyInit);
 5962
 5963        /* set iv */
 5964        iv = aes->reg;
 5965        CRYP_IVStructInit(&ivInit);
 5966        ByteReverseWords(iv, iv, WC_AES_BLOCK_SIZE);
 5967        ivInit.CRYP_IV0Left  = iv[0];
 5968        ivInit.CRYP_IV0Right = iv[1];
 5969        ivInit.CRYP_IV1Left  = iv[2];
 5970        ivInit.CRYP_IV1Right = iv[3];
 5971        CRYP_IVInit(&ivInit);
 5972
 5973        /* set direction and mode */
 5974        cryptInit.CRYP_AlgoDir  = CRYP_AlgoDir_Encrypt;
 5975        cryptInit.CRYP_AlgoMode = CRYP_AlgoMode_AES_CBC;
 5976        CRYP_Init(&cryptInit);
 5977
 5978        /* enable crypto processor */
 5979        CRYP_Cmd(ENABLE);
 5980
 5981        while (blocks--) {
 5982            /* flush IN/OUT FIFOs */
 5983            CRYP_FIFOFlush();
 5984
 5985            CRYP_DataIn(*(uint32_t*)&in[0]);
 5986            CRYP_DataIn(*(uint32_t*)&in[4]);
 5987            CRYP_DataIn(*(uint32_t*)&in[8]);
 5988            CRYP_DataIn(*(uint32_t*)&in[12]);
 5989
 5990            /* wait until the complete message has been processed */
 5991            while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
 5992
 5993            *(uint32_t*)&out[0]  = CRYP_DataOut();
 5994            *(uint32_t*)&out[4]  = CRYP_DataOut();
 5995            *(uint32_t*)&out[8]  = CRYP_DataOut();
 5996            *(uint32_t*)&out[12] = CRYP_DataOut();
 5997
 5998            /* store iv for next call */
 5999            XMEMCPY(aes->reg, out + sz - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
 6000
 6001            sz  -= WC_AES_BLOCK_SIZE;
 6002            in  += WC_AES_BLOCK_SIZE;
 6003            out += WC_AES_BLOCK_SIZE;
 6004        }
 6005
 6006        /* disable crypto processor */
 6007        CRYP_Cmd(DISABLE);
 6008        wolfSSL_CryptHwMutexUnLock();
 6009        wc_Stm32_Aes_Cleanup();
 6010
 6011        return ret;
 6012    }
 6013
 6014    #ifdef HAVE_AES_DECRYPT
 6015    int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 6016    {
 6017        int ret;
 6018        word32 *iv;
 6019        CRYP_InitTypeDef cryptInit;
 6020        CRYP_KeyInitTypeDef keyInit;
 6021        CRYP_IVInitTypeDef ivInit;
 6022        word32 blocks = (sz / WC_AES_BLOCK_SIZE);
 6023
 6024#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 6025        if (sz % WC_AES_BLOCK_SIZE) {
 6026            return BAD_LENGTH_E;
 6027        }
 6028#endif
 6029        if (blocks == 0)
 6030            return 0;
 6031
 6032        ret = wc_Stm32_Aes_Init(aes, &cryptInit, &keyInit);
 6033        if (ret != 0)
 6034            return ret;
 6035
 6036        ret = wolfSSL_CryptHwMutexLock();
 6037        if (ret != 0) {
 6038            return ret;
 6039        }
 6040
 6041        /* if input and output same will overwrite input iv */
 6042        XMEMCPY(aes->tmp, in + sz - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
 6043
 6044        /* reset registers to their default values */
 6045        CRYP_DeInit();
 6046
 6047        /* set direction and key */
 6048        CRYP_KeyInit(&keyInit);
 6049        cryptInit.CRYP_AlgoDir  = CRYP_AlgoDir_Decrypt;
 6050        cryptInit.CRYP_AlgoMode = CRYP_AlgoMode_AES_Key;
 6051        CRYP_Init(&cryptInit);
 6052
 6053        /* enable crypto processor */
 6054        CRYP_Cmd(ENABLE);
 6055
 6056        /* wait until key has been prepared */
 6057        while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
 6058
 6059        /* set direction and mode */
 6060        cryptInit.CRYP_AlgoDir  = CRYP_AlgoDir_Decrypt;
 6061        cryptInit.CRYP_AlgoMode = CRYP_AlgoMode_AES_CBC;
 6062        CRYP_Init(&cryptInit);
 6063
 6064        /* set iv */
 6065        iv = aes->reg;
 6066        CRYP_IVStructInit(&ivInit);
 6067        ByteReverseWords(iv, iv, WC_AES_BLOCK_SIZE);
 6068        ivInit.CRYP_IV0Left  = iv[0];
 6069        ivInit.CRYP_IV0Right = iv[1];
 6070        ivInit.CRYP_IV1Left  = iv[2];
 6071        ivInit.CRYP_IV1Right = iv[3];
 6072        CRYP_IVInit(&ivInit);
 6073
 6074        /* enable crypto processor */
 6075        CRYP_Cmd(ENABLE);
 6076
 6077        while (blocks--) {
 6078            /* flush IN/OUT FIFOs */
 6079            CRYP_FIFOFlush();
 6080
 6081            CRYP_DataIn(*(uint32_t*)&in[0]);
 6082            CRYP_DataIn(*(uint32_t*)&in[4]);
 6083            CRYP_DataIn(*(uint32_t*)&in[8]);
 6084            CRYP_DataIn(*(uint32_t*)&in[12]);
 6085
 6086            /* wait until the complete message has been processed */
 6087            while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
 6088
 6089            *(uint32_t*)&out[0]  = CRYP_DataOut();
 6090            *(uint32_t*)&out[4]  = CRYP_DataOut();
 6091            *(uint32_t*)&out[8]  = CRYP_DataOut();
 6092            *(uint32_t*)&out[12] = CRYP_DataOut();
 6093
 6094            /* store iv for next call */
 6095            XMEMCPY(aes->reg, aes->tmp, WC_AES_BLOCK_SIZE);
 6096
 6097            in  += WC_AES_BLOCK_SIZE;
 6098            out += WC_AES_BLOCK_SIZE;
 6099        }
 6100
 6101        /* disable crypto processor */
 6102        CRYP_Cmd(DISABLE);
 6103        wolfSSL_CryptHwMutexUnLock();
 6104        wc_Stm32_Aes_Cleanup();
 6105
 6106        return ret;
 6107    }
 6108    #endif /* HAVE_AES_DECRYPT */
 6109#endif /* WOLFSSL_STM32_CUBEMX */
 6110
 6111#elif defined(HAVE_COLDFIRE_SEC)
 6112    static WARN_UNUSED_RESULT int wc_AesCbcCrypt(
 6113        Aes* aes, byte* po, const byte* pi, word32 sz, word32 descHeader)
 6114    {
 6115        #ifdef DEBUG_WOLFSSL
 6116            int i; int stat1, stat2; int ret;
 6117        #endif
 6118
 6119        int size;
 6120        volatile int v;
 6121
 6122        if ((pi == NULL) || (po == NULL))
 6123            return BAD_FUNC_ARG;    /*wrong pointer*/
 6124
 6125#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 6126        if (sz % WC_AES_BLOCK_SIZE) {
 6127            return BAD_LENGTH_E;
 6128        }
 6129#endif
 6130
 6131        wc_LockMutex(&Mutex_AesSEC);
 6132
 6133        /* Set descriptor for SEC */
 6134        secDesc->length1 = 0x0;
 6135        secDesc->pointer1 = NULL;
 6136
 6137        secDesc->length2 = WC_AES_BLOCK_SIZE;
 6138        secDesc->pointer2 = (byte *)secReg; /* Initial Vector */
 6139
 6140        switch(aes->rounds) {
 6141            case 10: secDesc->length3 = 16; break;
 6142            case 12: secDesc->length3 = 24; break;
 6143            case 14: secDesc->length3 = 32; break;
 6144        }
 6145        XMEMCPY(secKey, aes->key, secDesc->length3);
 6146
 6147        secDesc->pointer3 = (byte *)secKey;
 6148        secDesc->pointer4 = AESBuffIn;
 6149        secDesc->pointer5 = AESBuffOut;
 6150        secDesc->length6 = 0x0;
 6151        secDesc->pointer6 = NULL;
 6152        secDesc->length7 = 0x0;
 6153        secDesc->pointer7 = NULL;
 6154        secDesc->nextDescriptorPtr = NULL;
 6155
 6156#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 6157        size = AES_BUFFER_SIZE;
 6158#endif
 6159        while (sz) {
 6160            secDesc->header = descHeader;
 6161            XMEMCPY(secReg, aes->reg, WC_AES_BLOCK_SIZE);
 6162#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 6163            sz -= AES_BUFFER_SIZE;
 6164#else
 6165            if (sz < AES_BUFFER_SIZE) {
 6166                size = sz;
 6167                sz = 0;
 6168            } else {
 6169                size = AES_BUFFER_SIZE;
 6170                sz -= AES_BUFFER_SIZE;
 6171            }
 6172#endif
 6173
 6174            secDesc->length4 = size;
 6175            secDesc->length5 = size;
 6176
 6177            XMEMCPY(AESBuffIn, pi, size);
 6178            if(descHeader == SEC_DESC_AES_CBC_DECRYPT) {
 6179                XMEMCPY((void*)aes->tmp, (void*)&(pi[size-WC_AES_BLOCK_SIZE]),
 6180                        WC_AES_BLOCK_SIZE);
 6181            }
 6182
 6183            /* Point SEC to the location of the descriptor */
 6184            MCF_SEC_FR0 = (uint32)secDesc;
 6185            /* Initialize SEC and wait for encryption to complete */
 6186            MCF_SEC_CCCR0 = 0x0000001a;
 6187            /* poll SISR to determine when channel is complete */
 6188            v=0;
 6189
 6190            while ((secDesc->header>> 24) != 0xff) v++;
 6191
 6192            #ifdef DEBUG_WOLFSSL
 6193                ret = MCF_SEC_SISRH;
 6194                stat1 = MCF_SEC_AESSR;
 6195                stat2 = MCF_SEC_AESISR;
 6196                if (ret & 0xe0000000) {
 6197                    db_printf("Aes_Cbc(i=%d):ISRH=%08x, AESSR=%08x, "
 6198                              "AESISR=%08x\n", i, ret, stat1, stat2);
 6199                }
 6200            #endif
 6201
 6202            XMEMCPY(po, AESBuffOut, size);
 6203
 6204            if (descHeader == SEC_DESC_AES_CBC_ENCRYPT) {
 6205                XMEMCPY((void*)aes->reg, (void*)&(po[size-WC_AES_BLOCK_SIZE]),
 6206                        WC_AES_BLOCK_SIZE);
 6207            } else {
 6208                XMEMCPY((void*)aes->reg, (void*)aes->tmp, WC_AES_BLOCK_SIZE);
 6209            }
 6210
 6211            pi += size;
 6212            po += size;
 6213        }
 6214
 6215        wc_UnLockMutex(&Mutex_AesSEC);
 6216        return 0;
 6217    }
 6218
 6219    int wc_AesCbcEncrypt(Aes* aes, byte* po, const byte* pi, word32 sz)
 6220    {
 6221        return (wc_AesCbcCrypt(aes, po, pi, sz, SEC_DESC_AES_CBC_ENCRYPT));
 6222    }
 6223
 6224    #ifdef HAVE_AES_DECRYPT
 6225    int wc_AesCbcDecrypt(Aes* aes, byte* po, const byte* pi, word32 sz)
 6226    {
 6227        return (wc_AesCbcCrypt(aes, po, pi, sz, SEC_DESC_AES_CBC_DECRYPT));
 6228    }
 6229    #endif /* HAVE_AES_DECRYPT */
 6230
 6231#elif defined(FREESCALE_LTC)
 6232    int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 6233    {
 6234        word32 keySize;
 6235        status_t status;
 6236        byte *iv, *enc_key;
 6237        word32 blocks = (sz / WC_AES_BLOCK_SIZE);
 6238
 6239#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 6240        if (sz % WC_AES_BLOCK_SIZE) {
 6241            return BAD_LENGTH_E;
 6242        }
 6243#endif
 6244        if (blocks == 0)
 6245            return 0;
 6246
 6247        iv      = (byte*)aes->reg;
 6248        enc_key = (byte*)aes->key;
 6249
 6250        status = wc_AesGetKeySize(aes, &keySize);
 6251        if (status != 0) {
 6252            return status;
 6253        }
 6254
 6255        status = wolfSSL_CryptHwMutexLock();
 6256        if (status != 0)
 6257            return status;
 6258        status = LTC_AES_EncryptCbc(LTC_BASE, in, out, blocks * WC_AES_BLOCK_SIZE,
 6259            iv, enc_key, keySize);
 6260        wolfSSL_CryptHwMutexUnLock();
 6261
 6262        /* store iv for next call */
 6263        if (status == kStatus_Success) {
 6264            XMEMCPY(iv, out + sz - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
 6265        }
 6266
 6267        return (status == kStatus_Success) ? 0 : -1;
 6268    }
 6269
 6270    #ifdef HAVE_AES_DECRYPT
 6271    int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 6272    {
 6273        word32 keySize;
 6274        status_t status;
 6275        byte* iv, *dec_key;
 6276        byte temp_block[WC_AES_BLOCK_SIZE];
 6277        word32 blocks = (sz / WC_AES_BLOCK_SIZE);
 6278
 6279#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 6280        if (sz % WC_AES_BLOCK_SIZE) {
 6281            return BAD_LENGTH_E;
 6282        }
 6283#endif
 6284        if (blocks == 0)
 6285            return 0;
 6286
 6287        iv      = (byte*)aes->reg;
 6288        dec_key = (byte*)aes->key;
 6289
 6290        status = wc_AesGetKeySize(aes, &keySize);
 6291        if (status != 0) {
 6292            return status;
 6293        }
 6294
 6295        /* get IV for next call */
 6296        XMEMCPY(temp_block, in + sz - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
 6297
 6298        status = wolfSSL_CryptHwMutexLock();
 6299        if (status != 0)
 6300            return status;
 6301        status = LTC_AES_DecryptCbc(LTC_BASE, in, out, blocks * WC_AES_BLOCK_SIZE,
 6302            iv, dec_key, keySize, kLTC_EncryptKey);
 6303        wolfSSL_CryptHwMutexUnLock();
 6304
 6305        /* store IV for next call */
 6306        if (status == kStatus_Success) {
 6307            XMEMCPY(iv, temp_block, WC_AES_BLOCK_SIZE);
 6308        }
 6309
 6310        return (status == kStatus_Success) ? 0 : -1;
 6311    }
 6312    #endif /* HAVE_AES_DECRYPT */
 6313
 6314#elif defined(FREESCALE_MMCAU) && !defined(WOLFSSL_ARMASM)
 6315    int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 6316    {
 6317        int offset = 0;
 6318        byte *iv;
 6319        byte temp_block[WC_AES_BLOCK_SIZE];
 6320        word32 blocks = (sz / WC_AES_BLOCK_SIZE);
 6321        int ret;
 6322
 6323#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 6324        if (sz % WC_AES_BLOCK_SIZE) {
 6325            return BAD_LENGTH_E;
 6326        }
 6327#endif
 6328        if (blocks == 0)
 6329            return 0;
 6330
 6331        iv = (byte*)aes->reg;
 6332
 6333        while (blocks--) {
 6334            XMEMCPY(temp_block, in + offset, WC_AES_BLOCK_SIZE);
 6335
 6336            /* XOR block with IV for CBC */
 6337            xorbuf(temp_block, iv, WC_AES_BLOCK_SIZE);
 6338
 6339            ret = wc_AesEncrypt(aes, temp_block, out + offset);
 6340            if (ret != 0)
 6341                return ret;
 6342
 6343            offset += WC_AES_BLOCK_SIZE;
 6344
 6345            /* store IV for next block */
 6346            XMEMCPY(iv, out + offset - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
 6347        }
 6348
 6349        return 0;
 6350    }
 6351    #ifdef HAVE_AES_DECRYPT
 6352    int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 6353    {
 6354        int ret;
 6355        int offset = 0;
 6356        byte* iv;
 6357        byte temp_block[WC_AES_BLOCK_SIZE];
 6358        word32 blocks = (sz / WC_AES_BLOCK_SIZE);
 6359
 6360#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 6361        if (sz % WC_AES_BLOCK_SIZE) {
 6362            return BAD_LENGTH_E;
 6363        }
 6364#endif
 6365        if (blocks == 0)
 6366            return 0;
 6367
 6368        iv = (byte*)aes->reg;
 6369
 6370        while (blocks--) {
 6371            XMEMCPY(temp_block, in + offset, WC_AES_BLOCK_SIZE);
 6372
 6373            ret = wc_AesDecrypt(aes, in + offset, out + offset);
 6374            if (ret != 0)
 6375                return ret;
 6376
 6377            /* XOR block with IV for CBC */
 6378            xorbuf(out + offset, iv, WC_AES_BLOCK_SIZE);
 6379
 6380            /* store IV for next block */
 6381            XMEMCPY(iv, temp_block, WC_AES_BLOCK_SIZE);
 6382
 6383            offset += WC_AES_BLOCK_SIZE;
 6384        }
 6385        return 0;
 6386    }
 6387    #endif /* HAVE_AES_DECRYPT */
 6388
 6389#elif defined(MAX3266X_AES)
 6390    int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 6391    {
 6392        word32 keySize;
 6393        int status;
 6394        byte *iv;
 6395
 6396        if ((in == NULL) || (out == NULL) || (aes == NULL)) {
 6397            return BAD_FUNC_ARG;
 6398        }
 6399
 6400        /* Always enforce a length check */
 6401        if (sz % WC_AES_BLOCK_SIZE) {
 6402        #ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 6403            return BAD_LENGTH_E;
 6404        #else
 6405            return BAD_FUNC_ARG;
 6406        #endif
 6407        }
 6408        if (sz == 0) {
 6409            return 0;
 6410        }
 6411
 6412        iv = (byte*)aes->reg;
 6413        status = wc_AesGetKeySize(aes, &keySize);
 6414        if (status != 0) {
 6415            return status;
 6416        }
 6417
 6418        status = wc_MXC_TPU_AesEncrypt(in, iv, (byte*)aes->key,
 6419                                        MXC_TPU_MODE_CBC, sz, out,
 6420                                        (unsigned int)keySize);
 6421        /* store iv for next call */
 6422        if (status == 0) {
 6423            XMEMCPY(iv, out + sz - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
 6424        }
 6425        return (status == 0) ? 0 : -1;
 6426    }
 6427
 6428    #ifdef HAVE_AES_DECRYPT
 6429    int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 6430    {
 6431        word32 keySize;
 6432        int status;
 6433        byte *iv;
 6434        byte temp_block[WC_AES_BLOCK_SIZE];
 6435
 6436        if ((in == NULL) || (out == NULL) || (aes == NULL)) {
 6437            return BAD_FUNC_ARG;
 6438        }
 6439
 6440        /* Always enforce a length check */
 6441        if (sz % WC_AES_BLOCK_SIZE) {
 6442        #ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 6443            return BAD_LENGTH_E;
 6444        #else
 6445            return BAD_FUNC_ARG;
 6446        #endif
 6447        }
 6448        if (sz == 0) {
 6449            return 0;
 6450        }
 6451
 6452        iv = (byte*)aes->reg;
 6453        status = wc_AesGetKeySize(aes, &keySize);
 6454        if (status != 0) {
 6455            return status;
 6456        }
 6457
 6458        /* get IV for next call */
 6459        XMEMCPY(temp_block, in + sz - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
 6460        status = wc_MXC_TPU_AesDecrypt(in, iv, (byte*)aes->key,
 6461                                        MXC_TPU_MODE_CBC, sz, out,
 6462                                        keySize);
 6463
 6464        /* store iv for next call */
 6465        if (status == 0) {
 6466            XMEMCPY(iv, temp_block, WC_AES_BLOCK_SIZE);
 6467        }
 6468        return (status == 0) ? 0 : -1;
 6469    }
 6470    #endif /* HAVE_AES_DECRYPT */
 6471
 6472
 6473
 6474#elif defined(WOLFSSL_PIC32MZ_CRYPT)
 6475
 6476    int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 6477    {
 6478        int ret;
 6479
 6480        if (sz == 0)
 6481            return 0;
 6482
 6483        /* hardware fails on input that is not a multiple of AES block size */
 6484        if (sz % WC_AES_BLOCK_SIZE != 0) {
 6485#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 6486            return BAD_LENGTH_E;
 6487#else
 6488            return BAD_FUNC_ARG;
 6489#endif
 6490        }
 6491
 6492        ret = wc_Pic32AesCrypt(
 6493            aes->key, aes->keylen, aes->reg, WC_AES_BLOCK_SIZE,
 6494            out, in, sz, PIC32_ENCRYPTION,
 6495            PIC32_ALGO_AES, PIC32_CRYPTOALGO_RCBC);
 6496
 6497        /* store iv for next call */
 6498        if (ret == 0) {
 6499            XMEMCPY(aes->reg, out + sz - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
 6500        }
 6501
 6502        return ret;
 6503    }
 6504    #ifdef HAVE_AES_DECRYPT
 6505    int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 6506    {
 6507        int ret;
 6508        byte scratch[WC_AES_BLOCK_SIZE];
 6509
 6510        if (sz == 0)
 6511            return 0;
 6512
 6513        /* hardware fails on input that is not a multiple of AES block size */
 6514        if (sz % WC_AES_BLOCK_SIZE != 0) {
 6515#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 6516            return BAD_LENGTH_E;
 6517#else
 6518            return BAD_FUNC_ARG;
 6519#endif
 6520        }
 6521        XMEMCPY(scratch, in + sz - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
 6522
 6523        ret = wc_Pic32AesCrypt(
 6524            aes->key, aes->keylen, aes->reg, WC_AES_BLOCK_SIZE,
 6525            out, in, sz, PIC32_DECRYPTION,
 6526            PIC32_ALGO_AES, PIC32_CRYPTOALGO_RCBC);
 6527
 6528        /* store iv for next call */
 6529        if (ret == 0) {
 6530            XMEMCPY((byte*)aes->reg, scratch, WC_AES_BLOCK_SIZE);
 6531        }
 6532
 6533        return ret;
 6534    }
 6535    #endif /* HAVE_AES_DECRYPT */
 6536#elif defined(WOLFSSL_ESP32_CRYPT) && \
 6537    !defined(NO_WOLFSSL_ESP32_CRYPT_AES)
 6538
 6539    /* We'll use SW for fall back:
 6540     *   unsupported key lengths
 6541     *   hardware busy */
 6542    #define NEED_SW_AESCBC
 6543    #define NEED_AESCBC_HW_FALLBACK
 6544
 6545#elif defined(WOLFSSL_CRYPTOCELL) && defined(WOLFSSL_CRYPTOCELL_AES)
 6546    int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 6547    {
 6548        return SaSi_AesBlock(&aes->ctx.user_ctx, (uint8_t*)in, sz, out);
 6549    }
 6550    int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 6551    {
 6552        return SaSi_AesBlock(&aes->ctx.user_ctx, (uint8_t*)in, sz, out);
 6553    }
 6554#elif defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES) && \
 6555        !defined(WOLFSSL_QNX_CAAM)
 6556      /* implemented in wolfcrypt/src/port/caam/caam_aes.c */
 6557
 6558#elif defined(WOLFSSL_AFALG)
 6559    /* implemented in wolfcrypt/src/port/af_alg/afalg_aes.c */
 6560
 6561#elif defined(WOLFSSL_KCAPI_AES) && !defined(WOLFSSL_NO_KCAPI_AES_CBC)
 6562    /* implemented in wolfcrypt/src/port/kcapi/kcapi_aes.c */
 6563
 6564#elif defined(WOLFSSL_DEVCRYPTO_CBC)
 6565    /* implemented in wolfcrypt/src/port/devcrypt/devcrypto_aes.c */
 6566
 6567#elif defined(WOLFSSL_NXP_HASHCRYPT_AES)
 6568    /* implemented in wolfcrypt/src/port/nxp/hashcrypt_port.c */
 6569
 6570#elif defined(WOLFSSL_SILABS_SE_ACCEL)
 6571    /* implemented in wolfcrypt/src/port/silabs/silabs_aes.c */
 6572
 6573#elif defined(WOLFSSL_HAVE_PSA) && !defined(WOLFSSL_PSA_NO_AES)
 6574    /* implemented in wolfcrypt/src/port/psa/psa_aes.c */
 6575
 6576#elif defined(WOLFSSL_PSOC6_CRYPTO)
 6577
 6578    int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 6579    {
 6580        return wc_Psoc6_Aes_CbcEncrypt(aes, out, in, sz);
 6581    }
 6582
 6583    #if defined(HAVE_AES_DECRYPT)
 6584    int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 6585    {
 6586        return wc_Psoc6_Aes_CbcDecrypt(aes, out, in, sz);
 6587    }
 6588    #endif /* HAVE_AES_DECRYPT */
 6589
 6590#else
 6591    /* Reminder: Some HW implementations may also define this as needed.
 6592     * (e.g. for unsupported key length fallback)  */
 6593    #define NEED_SW_AESCBC
 6594#endif
 6595
 6596#ifdef NEED_SW_AESCBC
 6597    /* Software AES - CBC Encrypt */
 6598
 6599int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 6600    {
 6601#if !defined(WOLFSSL_ARMASM)
 6602        word32 blocks;
 6603        int ret;
 6604#endif
 6605
 6606        if (aes == NULL || out == NULL || in == NULL) {
 6607            return BAD_FUNC_ARG;
 6608        }
 6609
 6610        if (sz == 0) {
 6611            return 0;
 6612        }
 6613
 6614#if !defined(WOLFSSL_ARMASM)
 6615        blocks = sz / WC_AES_BLOCK_SIZE;
 6616#endif
 6617#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 6618        if (sz % WC_AES_BLOCK_SIZE) {
 6619            WOLFSSL_ERROR_VERBOSE(BAD_LENGTH_E);
 6620            return BAD_LENGTH_E;
 6621        }
 6622#endif
 6623
 6624    #ifdef WOLFSSL_IMXRT_DCP
 6625        /* Implemented in wolfcrypt/src/port/nxp/dcp_port.c */
 6626        if (aes->keylen == 16)
 6627            return DCPAesCbcEncrypt(aes, out, in, sz);
 6628    #endif
 6629
 6630    #ifdef WOLF_CRYPTO_CB
 6631        #ifndef WOLF_CRYPTO_CB_FIND
 6632        if (aes->devId != INVALID_DEVID)
 6633        #endif
 6634        {
 6635            int crypto_cb_ret = wc_CryptoCb_AesCbcEncrypt(aes, out, in, sz);
 6636            if (crypto_cb_ret != WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE))
 6637                return crypto_cb_ret;
 6638            /* fall-through when unavailable */
 6639        }
 6640    #endif
 6641    #if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)
 6642        /* if async and byte count above threshold */
 6643        if (aes->asyncDev.marker == WOLFSSL_ASYNC_MARKER_AES &&
 6644                                                sz >= WC_ASYNC_THRESH_AES_CBC) {
 6645        #if defined(HAVE_CAVIUM)
 6646            return NitroxAesCbcEncrypt(aes, out, in, sz);
 6647        #elif defined(HAVE_INTEL_QA)
 6648            return IntelQaSymAesCbcEncrypt(&aes->asyncDev, out, in, sz,
 6649                (const byte*)aes->devKey, aes->keylen,
 6650                (byte*)aes->reg, WC_AES_BLOCK_SIZE);
 6651        #elif defined(WOLFSSL_ASYNC_CRYPT_SW)
 6652            if (wc_AsyncSwInit(&aes->asyncDev, ASYNC_SW_AES_CBC_ENCRYPT)) {
 6653                WC_ASYNC_SW* sw = &aes->asyncDev.sw;
 6654                sw->aes.aes = aes;
 6655                sw->aes.out = out;
 6656                sw->aes.in = in;
 6657                sw->aes.sz = sz;
 6658                return WC_PENDING_E;
 6659            }
 6660        #endif
 6661        }
 6662    #endif /* WOLFSSL_ASYNC_CRYPT */
 6663
 6664#if defined(WOLFSSL_ARMASM)
 6665#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
 6666    #if !defined(__aarch64__)
 6667        AES_CBC_encrypt_AARCH32(in, out, sz, (byte*)aes->reg, (byte*)aes->key,
 6668            (int)aes->rounds);
 6669    #else
 6670        if (aes->use_aes_hw_crypto) {
 6671            AES_CBC_encrypt_AARCH64(in, out, sz, (byte*)aes->reg,
 6672                (byte*)aes->key, (int)aes->rounds);
 6673        }
 6674        else
 6675    #endif /* __aarch64__ */
 6676#endif /* !WOLFSSL_ARMASM_NO_HW_CRYPTO */
 6677    #if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON) && \
 6678        defined(WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP)
 6679        {
 6680            AES_CBC_encrypt_NEON(in, out, sz, (const unsigned char*)aes->key,
 6681                aes->rounds, (unsigned char*)aes->reg);
 6682        }
 6683    #elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
 6684        {
 6685            AES_CBC_encrypt(in, out, sz, (const unsigned char*)aes->key,
 6686                aes->rounds, (unsigned char*)aes->reg);
 6687        }
 6688    #endif
 6689        return 0;
 6690#else
 6691    #if defined(WOLFSSL_SE050) && defined(WOLFSSL_SE050_CRYPT)
 6692        /* Implemented in wolfcrypt/src/port/nxp/se050_port.c */
 6693        if (aes->useSWCrypt == 0) {
 6694            return se050_aes_crypt(aes, in, out, sz, AES_ENCRYPTION,
 6695                                   kAlgorithm_SSS_AES_CBC);
 6696        }
 6697        else
 6698    #elif defined(WOLFSSL_ESPIDF) && defined(NEED_AESCBC_HW_FALLBACK)
 6699        if (wc_esp32AesSupportedKeyLen(aes)) {
 6700            ESP_LOGV(TAG, "wc_AesCbcEncrypt calling wc_esp32AesCbcEncrypt");
 6701            return wc_esp32AesCbcEncrypt(aes, out, in, sz);
 6702        }
 6703        else {
 6704            /* For example, the ESP32-S3 does not support HW for len = 24,
 6705             * so fall back to SW */
 6706        #ifdef DEBUG_WOLFSSL
 6707            ESP_LOGW(TAG, "wc_AesCbcEncrypt HW Falling back, "
 6708                          "unsupported keylen = %d", aes->keylen);
 6709        #endif
 6710        }
 6711    #elif defined(WOLFSSL_AESNI)
 6712        VECTOR_REGISTERS_PUSH;
 6713        if (aes->use_aesni) {
 6714            #ifdef DEBUG_AESNI
 6715                printf("about to aes cbc encrypt\n");
 6716                printf("in  = %p\n", in);
 6717                printf("out = %p\n", out);
 6718                printf("aes->key = %p\n", aes->key);
 6719                printf("aes->reg = %p\n", aes->reg);
 6720                printf("aes->rounds = %d\n", aes->rounds);
 6721                printf("sz = %d\n", sz);
 6722            #endif
 6723
 6724            /* check alignment, decrypt doesn't need alignment */
 6725            if ((wc_ptr_t)in % AESNI_ALIGN) {
 6726            #ifndef NO_WOLFSSL_ALLOC_ALIGN
 6727                byte* tmp = (byte*)XMALLOC(sz + WC_AES_BLOCK_SIZE + AESNI_ALIGN,
 6728                                            aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
 6729                byte* tmp_align;
 6730                if (tmp == NULL)
 6731                    ret = MEMORY_E;
 6732                else {
 6733                    tmp_align = tmp + (AESNI_ALIGN - ((wc_ptr_t)tmp % AESNI_ALIGN));
 6734                    XMEMCPY(tmp_align, in, sz);
 6735                    AES_CBC_encrypt_AESNI(tmp_align, tmp_align, (byte*)aes->reg, sz,
 6736                                          (byte*)aes->key, (int)aes->rounds);
 6737                    /* store iv for next call */
 6738                    XMEMCPY(aes->reg, tmp_align + sz - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
 6739
 6740                    XMEMCPY(out, tmp_align, sz);
 6741                    XFREE(tmp, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
 6742                    ret = 0;
 6743                }
 6744            #else
 6745                WOLFSSL_MSG("AES-CBC encrypt with bad alignment");
 6746                WOLFSSL_ERROR_VERBOSE(BAD_ALIGN_E);
 6747                ret = BAD_ALIGN_E;
 6748            #endif
 6749            } else {
 6750                AES_CBC_encrypt_AESNI(in, out, (byte*)aes->reg, sz, (byte*)aes->key,
 6751                                      (int)aes->rounds);
 6752                /* store iv for next call */
 6753                XMEMCPY(aes->reg, out + sz - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
 6754
 6755                ret = 0;
 6756            }
 6757        }
 6758        else
 6759    #endif
 6760        {
 6761#ifdef WC_AES_HAVE_PREFETCH_ARG
 6762            int did_prefetches = 0;
 6763#endif
 6764            ret = 0;
 6765            while (blocks--) {
 6766                xorbuf((byte*)aes->reg, in, WC_AES_BLOCK_SIZE);
 6767                ret = AesEncrypt_preFetchOpt(aes, (byte*)aes->reg,
 6768                                                (byte*)aes->reg,
 6769                                                &did_prefetches);
 6770                if (ret != 0)
 6771                    break;
 6772                XMEMCPY(out, aes->reg, WC_AES_BLOCK_SIZE);
 6773
 6774                out += WC_AES_BLOCK_SIZE;
 6775                in  += WC_AES_BLOCK_SIZE;
 6776            }
 6777        }
 6778
 6779    #ifdef WOLFSSL_AESNI
 6780        VECTOR_REGISTERS_POP;
 6781    #endif
 6782
 6783        return ret;
 6784#endif
 6785    } /* wc_AesCbcEncrypt */
 6786
 6787#ifdef HAVE_AES_DECRYPT
 6788    /* Software AES - CBC Decrypt */
 6789    int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 6790    {
 6791#if !defined(WOLFSSL_ARMASM)
 6792        word32 blocks;
 6793        int ret;
 6794#endif
 6795
 6796        if (aes == NULL || out == NULL || in == NULL) {
 6797            return BAD_FUNC_ARG;
 6798        }
 6799
 6800        if (sz == 0) {
 6801            return 0;
 6802        }
 6803
 6804    #if defined(WOLFSSL_ESPIDF) && defined(NEED_AESCBC_HW_FALLBACK)
 6805        if (wc_esp32AesSupportedKeyLen(aes)) {
 6806            ESP_LOGV(TAG, "wc_AesCbcDecrypt calling wc_esp32AesCbcDecrypt");
 6807            return wc_esp32AesCbcDecrypt(aes, out, in, sz);
 6808        }
 6809        else {
 6810            /* For example, the ESP32-S3 does not support HW for len = 24,
 6811             * so fall back to SW */
 6812        #ifdef DEBUG_WOLFSSL
 6813            ESP_LOGW(TAG, "wc_AesCbcDecrypt HW Falling back, "
 6814                          "unsupported keylen = %d", aes->keylen);
 6815        #endif
 6816        }
 6817    #endif
 6818
 6819#if !defined(WOLFSSL_ARMASM)
 6820        blocks = sz / WC_AES_BLOCK_SIZE;
 6821#endif
 6822        if (sz % WC_AES_BLOCK_SIZE) {
 6823#ifdef WOLFSSL_AES_CBC_LENGTH_CHECKS
 6824            return BAD_LENGTH_E;
 6825#else
 6826            return BAD_FUNC_ARG;
 6827#endif
 6828        }
 6829
 6830    #ifdef WOLFSSL_IMXRT_DCP
 6831        /* Implemented in wolfcrypt/src/port/nxp/dcp_port.c */
 6832        if (aes->keylen == 16)
 6833            return DCPAesCbcDecrypt(aes, out, in, sz);
 6834    #endif
 6835
 6836    #ifdef WOLF_CRYPTO_CB
 6837        #ifndef WOLF_CRYPTO_CB_FIND
 6838        if (aes->devId != INVALID_DEVID)
 6839        #endif
 6840        {
 6841            int crypto_cb_ret = wc_CryptoCb_AesCbcDecrypt(aes, out, in, sz);
 6842            if (crypto_cb_ret != WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE))
 6843                return crypto_cb_ret;
 6844            /* fall-through when unavailable */
 6845        }
 6846    #endif
 6847    #if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)
 6848        /* if async and byte count above threshold */
 6849        if (aes->asyncDev.marker == WOLFSSL_ASYNC_MARKER_AES &&
 6850                                                sz >= WC_ASYNC_THRESH_AES_CBC) {
 6851        #if defined(HAVE_CAVIUM)
 6852            return NitroxAesCbcDecrypt(aes, out, in, sz);
 6853        #elif defined(HAVE_INTEL_QA)
 6854            return IntelQaSymAesCbcDecrypt(&aes->asyncDev, out, in, sz,
 6855                (const byte*)aes->devKey, aes->keylen,
 6856                (byte*)aes->reg, WC_AES_BLOCK_SIZE);
 6857        #elif defined(WOLFSSL_ASYNC_CRYPT_SW)
 6858            if (wc_AsyncSwInit(&aes->asyncDev, ASYNC_SW_AES_CBC_DECRYPT)) {
 6859                WC_ASYNC_SW* sw = &aes->asyncDev.sw;
 6860                sw->aes.aes = aes;
 6861                sw->aes.out = out;
 6862                sw->aes.in = in;
 6863                sw->aes.sz = sz;
 6864                return WC_PENDING_E;
 6865            }
 6866        #endif
 6867        }
 6868    #endif
 6869
 6870    #if defined(WOLFSSL_SE050) && defined(WOLFSSL_SE050_CRYPT)
 6871        /* Implemented in wolfcrypt/src/port/nxp/se050_port.c */
 6872        if (aes->useSWCrypt == 0) {
 6873            return se050_aes_crypt(aes, in, out, sz, AES_DECRYPTION,
 6874                                   kAlgorithm_SSS_AES_CBC);
 6875        }
 6876    #endif
 6877
 6878#if defined(WOLFSSL_ARMASM)
 6879#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
 6880    #if !defined(__aarch64__)
 6881        AES_CBC_decrypt_AARCH32(in, out, sz, (byte*)aes->reg, (byte*)aes->key,
 6882            (int)aes->rounds);
 6883    #else
 6884        if (aes->use_aes_hw_crypto) {
 6885            AES_CBC_decrypt_AARCH64(in, out, sz, (byte*)aes->reg,
 6886                (byte*)aes->key, (int)aes->rounds);
 6887        }
 6888        else
 6889    #endif /* !__aarch64__ */
 6890#endif /* !WOLFSSL_ARMASM_NO_HW_CRYPTO */
 6891    #if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON)
 6892    #ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
 6893        if (sz >= 64)
 6894    #endif
 6895        {
 6896            AES_CBC_decrypt_NEON(in, out, sz, (const unsigned char*)aes->key,
 6897                aes->rounds, (unsigned char*)aes->reg);
 6898        }
 6899    #ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
 6900        else
 6901    #endif
 6902    #endif /* __aarch64__ || WOLFSSL_ARMASM_NO_HW_CRYPTO */
 6903    #if defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
 6904    #ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
 6905        {
 6906            AES_CBC_decrypt(in, out, sz, (const unsigned char*)aes->key,
 6907                aes->rounds, (unsigned char*)aes->reg);
 6908        }
 6909    #endif
 6910    #endif /* __aarch64__ || WOLFSSL_ARMASM_NO_HW_CRYPTO */
 6911        return 0;
 6912#else
 6913        VECTOR_REGISTERS_PUSH;
 6914
 6915    #ifdef WOLFSSL_AESNI
 6916        if (aes->use_aesni) {
 6917            #ifdef DEBUG_AESNI
 6918                printf("about to aes cbc decrypt\n");
 6919                printf("in  = %p\n", in);
 6920                printf("out = %p\n", out);
 6921                printf("aes->key = %p\n", aes->key);
 6922                printf("aes->reg = %p\n", aes->reg);
 6923                printf("aes->rounds = %d\n", aes->rounds);
 6924                printf("sz = %d\n", sz);
 6925            #endif
 6926
 6927            /* if input and output same will overwrite input iv */
 6928            XMEMCPY(aes->tmp, in + sz - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
 6929            #if defined(WOLFSSL_AESNI_BY4) || defined(WOLFSSL_X86_BUILD)
 6930            AES_CBC_decrypt_AESNI_by4(in, out, (byte*)aes->reg, sz, (byte*)aes->key,
 6931                            aes->rounds);
 6932            #elif defined(WOLFSSL_AESNI_BY6)
 6933            AES_CBC_decrypt_AESNI_by6(in, out, (byte*)aes->reg, sz, (byte*)aes->key,
 6934                            aes->rounds);
 6935            #else /* WOLFSSL_AESNI_BYx */
 6936            AES_CBC_decrypt_AESNI_by8(in, out, (byte*)aes->reg, sz, (byte*)aes->key,
 6937                            (int)aes->rounds);
 6938            #endif /* WOLFSSL_AESNI_BYx */
 6939            /* store iv for next call */
 6940            XMEMCPY(aes->reg, aes->tmp, WC_AES_BLOCK_SIZE);
 6941            ret = 0;
 6942        }
 6943        else
 6944    #endif
 6945        {
 6946            ret = 0;
 6947#ifdef WC_AES_BITSLICED
 6948            if (in != out) {
 6949                unsigned char dec[WC_AES_BLOCK_SIZE * BS_WORD_SIZE];
 6950
 6951                while (blocks > BS_WORD_SIZE) {
 6952                    AesDecryptBlocks_C(aes, in, dec, WC_AES_BLOCK_SIZE * BS_WORD_SIZE);
 6953                    xorbufout(out, dec, aes->reg, WC_AES_BLOCK_SIZE);
 6954                    xorbufout(out + WC_AES_BLOCK_SIZE, dec + WC_AES_BLOCK_SIZE, in,
 6955                              WC_AES_BLOCK_SIZE * (BS_WORD_SIZE - 1));
 6956                    XMEMCPY(aes->reg, in + (WC_AES_BLOCK_SIZE * (BS_WORD_SIZE - 1)),
 6957                            WC_AES_BLOCK_SIZE);
 6958                    in += WC_AES_BLOCK_SIZE * BS_WORD_SIZE;
 6959                    out += WC_AES_BLOCK_SIZE * BS_WORD_SIZE;
 6960                    blocks -= BS_WORD_SIZE;
 6961                }
 6962                if (blocks > 0) {
 6963                    AesDecryptBlocks_C(aes, in, dec, blocks * WC_AES_BLOCK_SIZE);
 6964                    xorbufout(out, dec, aes->reg, WC_AES_BLOCK_SIZE);
 6965                    xorbufout(out + WC_AES_BLOCK_SIZE, dec + WC_AES_BLOCK_SIZE, in,
 6966                              WC_AES_BLOCK_SIZE * (blocks - 1));
 6967                    XMEMCPY(aes->reg, in + (WC_AES_BLOCK_SIZE * (blocks - 1)),
 6968                            WC_AES_BLOCK_SIZE);
 6969                    blocks = 0;
 6970                }
 6971            }
 6972            else {
 6973                unsigned char dec[WC_AES_BLOCK_SIZE * BS_WORD_SIZE];
 6974                int i;
 6975
 6976                while (blocks > BS_WORD_SIZE) {
 6977                    AesDecryptBlocks_C(aes, in, dec, WC_AES_BLOCK_SIZE * BS_WORD_SIZE);
 6978                    XMEMCPY(aes->tmp, in + (BS_WORD_SIZE - 1) * WC_AES_BLOCK_SIZE,
 6979                            WC_AES_BLOCK_SIZE);
 6980                    for (i = BS_WORD_SIZE-1; i >= 1; i--) {
 6981                        xorbufout(out + i * WC_AES_BLOCK_SIZE,
 6982                                  dec + i * WC_AES_BLOCK_SIZE, in + (i - 1) * WC_AES_BLOCK_SIZE,
 6983                                  WC_AES_BLOCK_SIZE);
 6984                    }
 6985                    xorbufout(out, dec, aes->reg, WC_AES_BLOCK_SIZE);
 6986                    XMEMCPY(aes->reg, aes->tmp, WC_AES_BLOCK_SIZE);
 6987
 6988                    in += WC_AES_BLOCK_SIZE * BS_WORD_SIZE;
 6989                    out += WC_AES_BLOCK_SIZE * BS_WORD_SIZE;
 6990                    blocks -= BS_WORD_SIZE;
 6991                }
 6992                if (blocks > 0) {
 6993                    AesDecryptBlocks_C(aes, in, dec, blocks * WC_AES_BLOCK_SIZE);
 6994                    XMEMCPY(aes->tmp, in + (blocks - 1) * WC_AES_BLOCK_SIZE,
 6995                            WC_AES_BLOCK_SIZE);
 6996                    for (i = blocks-1; i >= 1; i--) {
 6997                        xorbufout(out + i * WC_AES_BLOCK_SIZE,
 6998                                  dec + i * WC_AES_BLOCK_SIZE, in + (i - 1) * WC_AES_BLOCK_SIZE,
 6999                                  WC_AES_BLOCK_SIZE);
 7000                    }
 7001                    xorbufout(out, dec, aes->reg, WC_AES_BLOCK_SIZE);
 7002                    XMEMCPY(aes->reg, aes->tmp, WC_AES_BLOCK_SIZE);
 7003
 7004                    blocks = 0;
 7005                }
 7006            }
 7007#else
 7008#ifdef WC_AES_HAVE_PREFETCH_ARG
 7009            {
 7010            int did_prefetches = 0;
 7011#endif
 7012            while (blocks--) {
 7013                XMEMCPY(aes->tmp, in, WC_AES_BLOCK_SIZE);
 7014                ret = AesDecrypt_preFetchOpt(aes, in, out, &did_prefetches);
 7015                if (ret != 0)
 7016                    return ret;
 7017                xorbuf(out, (byte*)aes->reg, WC_AES_BLOCK_SIZE);
 7018                /* store iv for next call */
 7019                XMEMCPY(aes->reg, aes->tmp, WC_AES_BLOCK_SIZE);
 7020
 7021                out += WC_AES_BLOCK_SIZE;
 7022                in  += WC_AES_BLOCK_SIZE;
 7023            }
 7024#ifdef WC_AES_HAVE_PREFETCH_ARG
 7025            }
 7026#endif
 7027#endif
 7028        }
 7029
 7030        VECTOR_REGISTERS_POP;
 7031
 7032        return ret;
 7033#endif
 7034    }
 7035#endif /* HAVE_AES_DECRYPT */
 7036
 7037#endif /* AES-CBC block */
 7038#endif /* HAVE_AES_CBC */
 7039
 7040/* AES-CTR */
 7041#if defined(WOLFSSL_AES_COUNTER)
 7042
 7043    #ifdef STM32_CRYPTO
 7044        #define NEED_AES_CTR_SOFT
 7045        #define XTRANSFORM_AESCTRBLOCK wc_AesCtrEncryptBlock
 7046
 7047        int wc_AesCtrEncryptBlock(Aes* aes, byte* out, const byte* in)
 7048        {
 7049            int ret = 0;
 7050        #ifdef WOLFSSL_STM32_CUBEMX
 7051            CRYP_HandleTypeDef hcryp;
 7052            #ifdef STM32_HAL_V2
 7053            word32 iv[WC_AES_BLOCK_SIZE/sizeof(word32)];
 7054            #endif
 7055        #else
 7056            word32 *iv;
 7057            CRYP_InitTypeDef cryptInit;
 7058            CRYP_KeyInitTypeDef keyInit;
 7059            CRYP_IVInitTypeDef ivInit;
 7060        #endif
 7061
 7062        #ifdef WOLFSSL_STM32_CUBEMX
 7063            ret = wc_Stm32_Aes_Init(aes, &hcryp, 0);
 7064            if (ret != 0) {
 7065                return ret;
 7066            }
 7067
 7068            ret = wolfSSL_CryptHwMutexLock();
 7069            if (ret != 0) {
 7070                return ret;
 7071            }
 7072
 7073        #if defined(STM32_HAL_V2)
 7074            hcryp.Init.Algorithm  = CRYP_AES_CTR;
 7075            ByteReverseWords(iv, aes->reg, WC_AES_BLOCK_SIZE);
 7076            hcryp.Init.pInitVect = (STM_CRYPT_TYPE*)iv;
 7077        #elif defined(STM32_CRYPTO_AES_ONLY)
 7078            hcryp.Init.OperatingMode = CRYP_ALGOMODE_ENCRYPT;
 7079            hcryp.Init.ChainingMode  = CRYP_CHAINMODE_AES_CTR;
 7080            hcryp.Init.KeyWriteFlag  = CRYP_KEY_WRITE_ENABLE;
 7081            hcryp.Init.pInitVect = (STM_CRYPT_TYPE*)aes->reg;
 7082        #else
 7083            hcryp.Init.pInitVect = (STM_CRYPT_TYPE*)aes->reg;
 7084        #endif
 7085            HAL_CRYP_Init(&hcryp);
 7086
 7087        #if defined(STM32_HAL_V2)
 7088            ret = HAL_CRYP_Encrypt(&hcryp, (uint32_t*)in, WC_AES_BLOCK_SIZE,
 7089                (uint32_t*)out, STM32_HAL_TIMEOUT);
 7090        #elif defined(STM32_CRYPTO_AES_ONLY)
 7091            ret = HAL_CRYPEx_AES(&hcryp, (byte*)in, WC_AES_BLOCK_SIZE,
 7092                out, STM32_HAL_TIMEOUT);
 7093        #else
 7094            ret = HAL_CRYP_AESCTR_Encrypt(&hcryp, (byte*)in, WC_AES_BLOCK_SIZE,
 7095                out, STM32_HAL_TIMEOUT);
 7096        #endif
 7097            if (ret != HAL_OK) {
 7098                ret = WC_TIMEOUT_E;
 7099            }
 7100            HAL_CRYP_DeInit(&hcryp);
 7101
 7102        #else /* Standard Peripheral Library */
 7103            ret = wc_Stm32_Aes_Init(aes, &cryptInit, &keyInit);
 7104            if (ret != 0) {
 7105                return ret;
 7106            }
 7107
 7108            ret = wolfSSL_CryptHwMutexLock();
 7109            if (ret != 0) {
 7110                return ret;
 7111            }
 7112
 7113            /* reset registers to their default values */
 7114            CRYP_DeInit();
 7115
 7116            /* set key */
 7117            CRYP_KeyInit(&keyInit);
 7118
 7119            /* set iv */
 7120            iv = aes->reg;
 7121            CRYP_IVStructInit(&ivInit);
 7122            ivInit.CRYP_IV0Left  = ByteReverseWord32(iv[0]);
 7123            ivInit.CRYP_IV0Right = ByteReverseWord32(iv[1]);
 7124            ivInit.CRYP_IV1Left  = ByteReverseWord32(iv[2]);
 7125            ivInit.CRYP_IV1Right = ByteReverseWord32(iv[3]);
 7126            CRYP_IVInit(&ivInit);
 7127
 7128            /* set direction and mode */
 7129            cryptInit.CRYP_AlgoDir  = CRYP_AlgoDir_Encrypt;
 7130            cryptInit.CRYP_AlgoMode = CRYP_AlgoMode_AES_CTR;
 7131            CRYP_Init(&cryptInit);
 7132
 7133            /* enable crypto processor */
 7134            CRYP_Cmd(ENABLE);
 7135
 7136            /* flush IN/OUT FIFOs */
 7137            CRYP_FIFOFlush();
 7138
 7139            CRYP_DataIn(*(uint32_t*)&in[0]);
 7140            CRYP_DataIn(*(uint32_t*)&in[4]);
 7141            CRYP_DataIn(*(uint32_t*)&in[8]);
 7142            CRYP_DataIn(*(uint32_t*)&in[12]);
 7143
 7144            /* wait until the complete message has been processed */
 7145            while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
 7146
 7147            *(uint32_t*)&out[0]  = CRYP_DataOut();
 7148            *(uint32_t*)&out[4]  = CRYP_DataOut();
 7149            *(uint32_t*)&out[8]  = CRYP_DataOut();
 7150            *(uint32_t*)&out[12] = CRYP_DataOut();
 7151
 7152            /* disable crypto processor */
 7153            CRYP_Cmd(DISABLE);
 7154        #endif /* WOLFSSL_STM32_CUBEMX */
 7155
 7156            wolfSSL_CryptHwMutexUnLock();
 7157            wc_Stm32_Aes_Cleanup();
 7158            return ret;
 7159        }
 7160
 7161
 7162    #elif defined(WOLFSSL_PIC32MZ_CRYPT)
 7163
 7164        #define NEED_AES_CTR_SOFT
 7165        #define XTRANSFORM_AESCTRBLOCK wc_AesCtrEncryptBlock
 7166
 7167        int wc_AesCtrEncryptBlock(Aes* aes, byte* out, const byte* in)
 7168        {
 7169            word32 tmpIv[WC_AES_BLOCK_SIZE / sizeof(word32)];
 7170            XMEMCPY(tmpIv, aes->reg, WC_AES_BLOCK_SIZE);
 7171            return wc_Pic32AesCrypt(
 7172                aes->key, aes->keylen, tmpIv, WC_AES_BLOCK_SIZE,
 7173                out, in, WC_AES_BLOCK_SIZE,
 7174                PIC32_ENCRYPTION, PIC32_ALGO_AES, PIC32_CRYPTOALGO_RCTR);
 7175        }
 7176
 7177    #elif defined(HAVE_COLDFIRE_SEC)
 7178        #error "Coldfire SEC doesn't currently support AES-CTR mode"
 7179
 7180    #elif defined(FREESCALE_LTC)
 7181        int wc_AesCtrEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 7182        {
 7183            int ret = 0;
 7184            word32 keySize;
 7185            byte *iv, *enc_key;
 7186            byte* tmp;
 7187
 7188            if (aes == NULL || out == NULL || in == NULL) {
 7189                return BAD_FUNC_ARG;
 7190            }
 7191
 7192            /* consume any unused bytes left in aes->tmp */
 7193            tmp = (byte*)aes->tmp + WC_AES_BLOCK_SIZE - aes->left;
 7194            while (aes->left && sz) {
 7195                *(out++) = *(in++) ^ *(tmp++);
 7196                aes->left--;
 7197                sz--;
 7198            }
 7199
 7200            if (sz) {
 7201                iv      = (byte*)aes->reg;
 7202                enc_key = (byte*)aes->key;
 7203
 7204                ret = wc_AesGetKeySize(aes, &keySize);
 7205                if (ret != 0)
 7206                    return ret;
 7207
 7208                ret = wolfSSL_CryptHwMutexLock();
 7209                if (ret != 0)
 7210                    return ret;
 7211                LTC_AES_CryptCtr(LTC_BASE, in, out, sz,
 7212                    iv, enc_key, keySize, (byte*)aes->tmp,
 7213                    (uint32_t*)&aes->left);
 7214                wolfSSL_CryptHwMutexUnLock();
 7215            }
 7216
 7217            return ret;
 7218        }
 7219
 7220    #elif defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES) && \
 7221        !defined(WOLFSSL_QNX_CAAM)
 7222        /* implemented in wolfcrypt/src/port/caam/caam_aes.c */
 7223
 7224    #elif defined(WOLFSSL_AFALG)
 7225        /* implemented in wolfcrypt/src/port/af_alg/afalg_aes.c */
 7226
 7227    #elif defined(WOLFSSL_DEVCRYPTO_AES)
 7228        /* implemented in wolfcrypt/src/port/devcrypt/devcrypto_aes.c */
 7229
 7230    #elif defined(WOLFSSL_ESP32_CRYPT) && \
 7231        !defined(NO_WOLFSSL_ESP32_CRYPT_AES)
 7232        /* esp32 doesn't support CRT mode by hw.     */
 7233        /* use aes ecnryption plus sw implementation */
 7234        #define NEED_AES_CTR_SOFT
 7235
 7236    #elif defined(WOLFSSL_HAVE_PSA) && !defined(WOLFSSL_PSA_NO_AES)
 7237        /* implemented in wolfcrypt/src/port/psa/psa_aes.c */
 7238
 7239    #elif defined(WOLFSSL_NXP_HASHCRYPT_AES)
 7240        /* implemented in wolfcrypt/src/port/nxp/hashcrypt_port.c */
 7241
 7242    #else
 7243
 7244        /* Use software based AES counter */
 7245        #define NEED_AES_CTR_SOFT
 7246    #endif
 7247
 7248    #ifdef NEED_AES_CTR_SOFT
 7249        #ifndef WOLFSSL_ARMASM
 7250        /* Increment AES counter */
 7251        static WC_INLINE void IncrementAesCounter(byte* inOutCtr)
 7252        {
 7253            /* in network byte order so start at end and work back */
 7254            int i;
 7255            for (i = WC_AES_BLOCK_SIZE - 1; i >= 0; i--) {
 7256                if (++inOutCtr[i])  /* we're done unless we overflow */
 7257                    return;
 7258            }
 7259        }
 7260        #endif
 7261
 7262        /* Software AES - CTR Encrypt */
 7263        int wc_AesCtrEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
 7264        {
 7265    #if !(!defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
 7266          !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO))
 7267            byte scratch[WC_AES_BLOCK_SIZE];
 7268    #endif
 7269    #if !defined(WOLFSSL_ARMASM)
 7270            int ret = 0;
 7271    #endif
 7272            word32 processed;
 7273#ifdef WC_AES_HAVE_PREFETCH_ARG
 7274            int did_prefetches = 0;
 7275#endif
 7276
 7277    #if !(!defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
 7278          !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO))
 7279            XMEMSET(scratch, 0, sizeof(scratch));
 7280    #endif
 7281
 7282            if (aes == NULL || out == NULL || in == NULL) {
 7283                return BAD_FUNC_ARG;
 7284            }
 7285
 7286        #ifdef WOLF_CRYPTO_CB
 7287            #ifndef WOLF_CRYPTO_CB_FIND
 7288            if (aes->devId != INVALID_DEVID)
 7289            #endif
 7290            {
 7291                int crypto_cb_ret = wc_CryptoCb_AesCtrEncrypt(aes, out, in, sz);
 7292                if (crypto_cb_ret != WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE))
 7293                    return crypto_cb_ret;
 7294                /* fall-through when unavailable */
 7295            }
 7296        #endif
 7297
 7298            /* consume any unused bytes left in aes->tmp */
 7299            processed = min(aes->left, sz);
 7300            xorbufout(out, in, (byte*)aes->tmp + WC_AES_BLOCK_SIZE - aes->left,
 7301                      processed);
 7302            out += processed;
 7303            in += processed;
 7304            aes->left -= processed;
 7305            sz -= processed;
 7306
 7307    #if defined(WOLFSSL_ARMASM)
 7308        #ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
 7309            #ifndef __aarch64__
 7310            AES_CTR_encrypt_AARCH32(in, out, sz, (byte*)aes->reg,
 7311                (byte*)aes->key, (byte*)aes->tmp, &aes->left, aes->rounds);
 7312            #else
 7313            if (aes->use_aes_hw_crypto) {
 7314                AES_CTR_encrypt_AARCH64(in, out, sz, (byte*)aes->reg,
 7315                    (byte*)aes->key, (byte*)aes->tmp, &aes->left, aes->rounds);
 7316                return 0;
 7317            }
 7318            else
 7319            #endif /* !__aarch64__ */
 7320        #endif /* !WOLFSSL_ARMASM_NO_HW_CRYPTO */
 7321        #if defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
 7322            {
 7323                word32 numBlocks;
 7324                byte* tmp = (byte*)aes->tmp + WC_AES_BLOCK_SIZE - aes->left;
 7325                /* consume any unused bytes left in aes->tmp */
 7326                while ((aes->left != 0) && (sz != 0)) {
 7327                   *(out++) = *(in++) ^ *(tmp++);
 7328                   aes->left--;
 7329                   sz--;
 7330                }
 7331
 7332                /* do as many block size ops as possible */
 7333                numBlocks = sz / WC_AES_BLOCK_SIZE;
 7334                if (numBlocks > 0) {
 7335                #if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON)
 7336                #ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
 7337                    if (sz >= 32)
 7338                #endif
 7339                    {
 7340                        AES_CTR_encrypt_NEON(in, out,
 7341                            numBlocks * WC_AES_BLOCK_SIZE, (byte*)aes->key,
 7342                            aes->rounds, (byte*)aes->reg);
 7343                    }
 7344                #ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
 7345                    else
 7346                #endif
 7347                #endif
 7348                #ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
 7349                    {
 7350                        AES_CTR_encrypt(in, out, numBlocks * WC_AES_BLOCK_SIZE,
 7351                            (byte*)aes->key, aes->rounds, (byte*)aes->reg);
 7352                    }
 7353                #endif
 7354
 7355                    sz  -= numBlocks * WC_AES_BLOCK_SIZE;
 7356                    out += numBlocks * WC_AES_BLOCK_SIZE;
 7357                    in  += numBlocks * WC_AES_BLOCK_SIZE;
 7358                }
 7359
 7360                /* handle non block size remaining */
 7361                if (sz) {
 7362                    byte zeros[WC_AES_BLOCK_SIZE] = { 0, 0, 0, 0, 0, 0, 0, 0,
 7363                                                      0, 0, 0, 0, 0, 0, 0, 0 };
 7364
 7365                #if defined(__aarch64__) && \
 7366                    !defined(WOLFSSL_ARMASM_NO_NEON) && \
 7367                    defined(WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP)
 7368                    {
 7369                        AES_CTR_encrypt_NEON(zeros, (byte*)aes->tmp,
 7370                            WC_AES_BLOCK_SIZE, (byte*)aes->key, aes->rounds,
 7371                            (byte*)aes->reg);
 7372                    }
 7373                #else
 7374                    {
 7375                        AES_CTR_encrypt(zeros, (byte*)aes->tmp,
 7376                            WC_AES_BLOCK_SIZE, (byte*)aes->key, aes->rounds,
 7377                            (byte*)aes->reg);
 7378                    }
 7379                #endif
 7380
 7381                    aes->left = WC_AES_BLOCK_SIZE;
 7382                    tmp = (byte*)aes->tmp;
 7383
 7384                    while (sz--) {
 7385                        *(out++) = *(in++) ^ *(tmp++);
 7386                        aes->left--;
 7387                    }
 7388                }
 7389            }
 7390        #endif /* __aarch64__ || WOLFSSL_ARMASM_NO_HW_CRYPTO */
 7391            return 0;
 7392    #else
 7393            VECTOR_REGISTERS_PUSH;
 7394
 7395        #if defined(HAVE_AES_ECB) && !defined(WOLFSSL_PIC32MZ_CRYPT) && \
 7396            !defined(XTRANSFORM_AESCTRBLOCK)
 7397            if (in != out && sz >= WC_AES_BLOCK_SIZE) {
 7398                word32 blocks = sz / WC_AES_BLOCK_SIZE;
 7399                byte* counter = (byte*)aes->reg;
 7400                byte* c = out;
 7401                while (blocks--) {
 7402                    XMEMCPY(c, counter, WC_AES_BLOCK_SIZE);
 7403                    c += WC_AES_BLOCK_SIZE;
 7404                    IncrementAesCounter(counter);
 7405                }
 7406
 7407                /* reset number of blocks and then do encryption */
 7408                blocks = sz / WC_AES_BLOCK_SIZE;
 7409                wc_AesEcbEncrypt(aes, out, out, WC_AES_BLOCK_SIZE * blocks);
 7410                xorbuf(out, in, WC_AES_BLOCK_SIZE * blocks);
 7411                in += WC_AES_BLOCK_SIZE * blocks;
 7412                out += WC_AES_BLOCK_SIZE * blocks;
 7413                sz -= blocks * WC_AES_BLOCK_SIZE;
 7414            }
 7415            else
 7416        #endif
 7417            {
 7418            #ifdef WOLFSSL_CHECK_MEM_ZERO
 7419                wc_MemZero_Add("wc_AesCtrEncrypt scratch", scratch,
 7420                    WC_AES_BLOCK_SIZE);
 7421            #endif
 7422                /* do as many block size ops as possible */
 7423                while (sz >= WC_AES_BLOCK_SIZE) {
 7424                #ifdef XTRANSFORM_AESCTRBLOCK
 7425                    XTRANSFORM_AESCTRBLOCK(aes, out, in);
 7426                #else
 7427                    ret = AesEncrypt_preFetchOpt(aes, (byte*)aes->reg,
 7428                                                    scratch,
 7429                                                    &did_prefetches);
 7430                    if (ret != 0)
 7431                        break;
 7432                    xorbuf(scratch, in, WC_AES_BLOCK_SIZE);
 7433                    XMEMCPY(out, scratch, WC_AES_BLOCK_SIZE);
 7434                #endif
 7435                    IncrementAesCounter((byte*)aes->reg);
 7436
 7437                    out += WC_AES_BLOCK_SIZE;
 7438                    in  += WC_AES_BLOCK_SIZE;
 7439                    sz  -= WC_AES_BLOCK_SIZE;
 7440                    aes->left = 0;
 7441                }
 7442                ForceZero(scratch, WC_AES_BLOCK_SIZE);
 7443            }
 7444
 7445            /* handle non block size remaining and store unused byte count in left */
 7446            if ((ret == 0) && sz) {
 7447                ret = AesEncrypt_preFetchOpt(aes, (byte*)aes->reg,
 7448                                                (byte*)aes->tmp,
 7449                                                &did_prefetches);
 7450                if (ret == 0) {
 7451                    IncrementAesCounter((byte*)aes->reg);
 7452                    aes->left = WC_AES_BLOCK_SIZE - sz;
 7453                    xorbufout(out, in, aes->tmp, sz);
 7454                }
 7455            }
 7456
 7457            if (ret < 0)
 7458                ForceZero(scratch, WC_AES_BLOCK_SIZE);
 7459
 7460        #ifdef WOLFSSL_CHECK_MEM_ZERO
 7461            wc_MemZero_Check(scratch, WC_AES_BLOCK_SIZE);
 7462        #endif
 7463
 7464            VECTOR_REGISTERS_POP;
 7465
 7466            return ret;
 7467    #endif
 7468        }
 7469
 7470        int wc_AesCtrSetKey(Aes* aes, const byte* key, word32 len,
 7471                                        const byte* iv, int dir)
 7472        {
 7473            if (aes == NULL) {
 7474                return BAD_FUNC_ARG;
 7475            }
 7476            if (len > sizeof(aes->key)) {
 7477                return BAD_FUNC_ARG;
 7478            }
 7479
 7480            return wc_AesSetKey(aes, key, len, iv, dir);
 7481        }
 7482
 7483    #endif /* NEED_AES_CTR_SOFT */
 7484
 7485#endif /* WOLFSSL_AES_COUNTER */
 7486
 7487#ifndef WC_AES_HAVE_PREFETCH_ARG
 7488    #ifndef AesEncrypt_preFetchOpt
 7489        #define AesEncrypt_preFetchOpt(aes, inBlock, outBlock, do_preFetch) \
 7490            wc_AesEncrypt(aes, inBlock, outBlock)
 7491    #endif
 7492    #ifndef AesDecrypt_preFetchOpt
 7493        #define AesDecrypt_preFetchOpt(aes, inBlock, outBlock, do_preFetch) \
 7494            wc_AesDecrypt(aes, inBlock, outBlock)
 7495    #endif
 7496#endif
 7497
 7498#else  /* WOLFSSL_RISCV_ASM */
 7499
 7500#define AesEncrypt_preFetchOpt(aes, inBlock, outBlock, do_preFetch) \
 7501    wc_AesEncryptDirect(aes, outBlock, inBlock)
 7502#define AesDecrypt_preFetchOpt(aes, inBlock, outBlock, do_preFetch) \
 7503    wc_AesDecryptDirect(aes, outBlock, inBlock)
 7504
 7505#endif /* WOLFSSL_RISCV_ASM */
 7506
 7507/*
 7508 * The IV for AES GCM and CCM, stored in struct Aes's member reg, is comprised
 7509 * of two parts in order:
 7510 *   1. The fixed field which may be 0 or 4 bytes long. In TLS, this is set
 7511 *      to the implicit IV.
 7512 *   2. The explicit IV is generated by wolfCrypt. It needs to be managed
 7513 *      by wolfCrypt to ensure the IV is unique for each call to encrypt.
 7514 * The IV may be a 96-bit random value, or the 32-bit fixed value and a
 7515 * 64-bit set of 0 or random data. The final 32-bits of reg is used as a
 7516 * block counter during the encryption.
 7517 */
 7518
 7519#if (defined(HAVE_AESGCM) && !defined(WC_NO_RNG)) || defined(HAVE_AESCCM)
 7520static WC_INLINE void IncCtr(byte* ctr, word32 ctrSz)
 7521{
 7522    int i;
 7523    for (i = (int)ctrSz - 1; i >= 0; i--) {
 7524        if (++ctr[i])
 7525            break;
 7526    }
 7527}
 7528#endif /* HAVE_AESGCM || HAVE_AESCCM */
 7529
 7530
 7531#ifdef HAVE_AESGCM
 7532
 7533#ifdef WOLFSSL_AESGCM_STREAM
 7534    /* Access initialization counter data. */
 7535    #define AES_INITCTR(aes)        ((aes)->streamData + 0 * WC_AES_BLOCK_SIZE)
 7536    /* Access counter data. */
 7537    #define AES_COUNTER(aes)        ((aes)->streamData + 1 * WC_AES_BLOCK_SIZE)
 7538    /* Access tag data. */
 7539    #define AES_TAG(aes)            ((aes)->streamData + 2 * WC_AES_BLOCK_SIZE)
 7540    /* Access last GHASH block. */
 7541    #define AES_LASTGBLOCK(aes)     ((aes)->streamData + 3 * WC_AES_BLOCK_SIZE)
 7542    /* Access last encrypted block. */
 7543    #define AES_LASTBLOCK(aes)      ((aes)->streamData + 4 * WC_AES_BLOCK_SIZE)
 7544
 7545    #define GHASH_ONE_BLOCK     GHASH_ONE_BLOCK_SW
 7546#endif
 7547
 7548#if defined(HAVE_COLDFIRE_SEC)
 7549    #error "Coldfire SEC doesn't currently support AES-GCM mode"
 7550
 7551#endif
 7552
 7553#if defined(WOLFSSL_RISCV_ASM)
 7554    /* implemented in wolfcrypt/src/port/risc-v/riscv-64-aes.c */
 7555
 7556#elif defined(WOLFSSL_AFALG)
 7557    /* implemented in wolfcrypt/src/port/afalg/afalg_aes.c */
 7558
 7559#elif defined(WOLFSSL_KCAPI_AES)
 7560    /* implemented in wolfcrypt/src/port/kcapi/kcapi_aes.c */
 7561
 7562#elif defined(WOLFSSL_DEVCRYPTO_AES)
 7563    /* implemented in wolfcrypt/src/port/devcrypt/devcrypto_aes.c */
 7564
 7565#else /* software + AESNI implementation */
 7566
 7567#if !defined(FREESCALE_LTC_AES_GCM)
 7568#if (!(defined(__aarch64__) && defined(WOLFSSL_ARMASM))) || \
 7569    defined(WOLFSSL_AESGCM_STREAM)
 7570static WC_INLINE void IncrementGcmCounter(byte* inOutCtr)
 7571{
 7572    int i;
 7573
 7574    /* in network byte order so start at end and work back */
 7575    for (i = WC_AES_BLOCK_SIZE - 1; i >= WC_AES_BLOCK_SIZE - CTR_SZ; i--) {
 7576        if (++inOutCtr[i])  /* we're done unless we overflow */
 7577            return;
 7578    }
 7579}
 7580#endif
 7581#endif /* !FREESCALE_LTC_AES_GCM */
 7582
 7583#if !defined(WOLFSSL_ARMASM) || defined(__aarch64__) || \
 7584    defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
 7585#if defined(GCM_SMALL) || defined(GCM_TABLE) || defined(GCM_TABLE_4BIT)
 7586
 7587static WC_INLINE void FlattenSzInBits(byte* buf, word32 sz)
 7588{
 7589    /* Multiply the sz by 8 */
 7590    word32 szHi = (sz >> (8*sizeof(sz) - 3));
 7591    sz <<= 3;
 7592
 7593    /* copy over the words of the sz into the destination buffer */
 7594    buf[0] = (byte)(szHi >> 24);
 7595    buf[1] = (byte)(szHi >> 16);
 7596    buf[2] = (byte)(szHi >>  8);
 7597    buf[3] = (byte)szHi;
 7598    buf[4] = (byte)(sz >> 24);
 7599    buf[5] = (byte)(sz >> 16);
 7600    buf[6] = (byte)(sz >>  8);
 7601    buf[7] = (byte)sz;
 7602}
 7603
 7604
 7605static WC_INLINE void RIGHTSHIFTX(byte* x)
 7606{
 7607    int i;
 7608    int carryIn = 0;
 7609    volatile byte borrow = (byte)((0x00U - (x[15] & 0x01U)) & 0xE1U);
 7610
 7611    for (i = 0; i < WC_AES_BLOCK_SIZE; i++) {
 7612        int carryOut = (x[i] & 0x01) << 7;
 7613        x[i] = (byte) ((x[i] >> 1) | carryIn);
 7614        carryIn = carryOut;
 7615    }
 7616    x[0] ^= borrow;
 7617}
 7618
 7619#endif /* defined(GCM_SMALL) || defined(GCM_TABLE) || defined(GCM_TABLE_4BIT) */
 7620
 7621
 7622#ifdef GCM_TABLE
 7623
 7624void GenerateM0(Gcm* gcm)
 7625{
 7626    int i, j;
 7627    byte (*m)[WC_AES_BLOCK_SIZE] = gcm->M0;
 7628
 7629    XMEMCPY(m[128], gcm->H, WC_AES_BLOCK_SIZE);
 7630
 7631    for (i = 64; i > 0; i /= 2) {
 7632        XMEMCPY(m[i], m[i*2], WC_AES_BLOCK_SIZE);
 7633        RIGHTSHIFTX(m[i]);
 7634    }
 7635
 7636    for (i = 2; i < 256; i *= 2) {
 7637        for (j = 1; j < i; j++) {
 7638            XMEMCPY(m[i+j], m[i], WC_AES_BLOCK_SIZE);
 7639            xorbuf(m[i+j], m[j], WC_AES_BLOCK_SIZE);
 7640        }
 7641    }
 7642
 7643    XMEMSET(m[0], 0, WC_AES_BLOCK_SIZE);
 7644}
 7645
 7646#elif defined(GCM_TABLE_4BIT)
 7647
 7648#if !defined(WC_16BIT_CPU)
 7649static WC_INLINE void Shift4_M0(byte *r8, byte *z8)
 7650{
 7651    int i;
 7652    for (i = 15; i > 0; i--)
 7653        r8[i] = (byte)(z8[i-1] << 4) | (byte)(z8[i] >> 4);
 7654    r8[0] = (byte)(z8[0] >> 4);
 7655}
 7656#endif
 7657
 7658void GenerateM0(Gcm* gcm)
 7659{
 7660#if !defined(WC_16BIT_CPU)
 7661    int i;
 7662#endif
 7663    byte (*m)[WC_AES_BLOCK_SIZE] = gcm->M0;
 7664
 7665    /* 0 times -> 0x0 */
 7666    XMEMSET(m[0x0], 0, WC_AES_BLOCK_SIZE);
 7667    /* 1 times -> 0x8 */
 7668    XMEMCPY(m[0x8], gcm->H, WC_AES_BLOCK_SIZE);
 7669    /* 2 times -> 0x4 */
 7670    XMEMCPY(m[0x4], m[0x8], WC_AES_BLOCK_SIZE);
 7671    RIGHTSHIFTX(m[0x4]);
 7672    /* 4 times -> 0x2 */
 7673    XMEMCPY(m[0x2], m[0x4], WC_AES_BLOCK_SIZE);
 7674    RIGHTSHIFTX(m[0x2]);
 7675    /* 8 times -> 0x1 */
 7676    XMEMCPY(m[0x1], m[0x2], WC_AES_BLOCK_SIZE);
 7677    RIGHTSHIFTX(m[0x1]);
 7678
 7679    /* 0x3 */
 7680    XMEMCPY(m[0x3], m[0x2], WC_AES_BLOCK_SIZE);
 7681    xorbuf (m[0x3], m[0x1], WC_AES_BLOCK_SIZE);
 7682
 7683    /* 0x5 -> 0x7 */
 7684    XMEMCPY(m[0x5], m[0x4], WC_AES_BLOCK_SIZE);
 7685    xorbuf (m[0x5], m[0x1], WC_AES_BLOCK_SIZE);
 7686    XMEMCPY(m[0x6], m[0x4], WC_AES_BLOCK_SIZE);
 7687    xorbuf (m[0x6], m[0x2], WC_AES_BLOCK_SIZE);
 7688    XMEMCPY(m[0x7], m[0x4], WC_AES_BLOCK_SIZE);
 7689    xorbuf (m[0x7], m[0x3], WC_AES_BLOCK_SIZE);
 7690
 7691    /* 0x9 -> 0xf */
 7692    XMEMCPY(m[0x9], m[0x8], WC_AES_BLOCK_SIZE);
 7693    xorbuf (m[0x9], m[0x1], WC_AES_BLOCK_SIZE);
 7694    XMEMCPY(m[0xa], m[0x8], WC_AES_BLOCK_SIZE);
 7695    xorbuf (m[0xa], m[0x2], WC_AES_BLOCK_SIZE);
 7696    XMEMCPY(m[0xb], m[0x8], WC_AES_BLOCK_SIZE);
 7697    xorbuf (m[0xb], m[0x3], WC_AES_BLOCK_SIZE);
 7698    XMEMCPY(m[0xc], m[0x8], WC_AES_BLOCK_SIZE);
 7699    xorbuf (m[0xc], m[0x4], WC_AES_BLOCK_SIZE);
 7700    XMEMCPY(m[0xd], m[0x8], WC_AES_BLOCK_SIZE);
 7701    xorbuf (m[0xd], m[0x5], WC_AES_BLOCK_SIZE);
 7702    XMEMCPY(m[0xe], m[0x8], WC_AES_BLOCK_SIZE);
 7703    xorbuf (m[0xe], m[0x6], WC_AES_BLOCK_SIZE);
 7704    XMEMCPY(m[0xf], m[0x8], WC_AES_BLOCK_SIZE);
 7705    xorbuf (m[0xf], m[0x7], WC_AES_BLOCK_SIZE);
 7706
 7707#if !defined(WC_16BIT_CPU)
 7708    for (i = 0; i < 16; i++) {
 7709        Shift4_M0(m[16+i], m[i]);
 7710    }
 7711#endif
 7712
 7713#if defined(WOLFSSL_ARMASM) && defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
 7714    for (i = 0; i < 32; i++) {
 7715    #if !defined(__aarch64__)
 7716        word32* m32 = (word32*)gcm->M0[i];
 7717        m32[0] = ByteReverseWord32(m32[0]);
 7718        m32[1] = ByteReverseWord32(m32[1]);
 7719        m32[2] = ByteReverseWord32(m32[2]);
 7720        m32[3] = ByteReverseWord32(m32[3]);
 7721    #else
 7722        word64* m64 = (word64*)gcm->M0[i];
 7723        m64[0] = ByteReverseWord64(m64[0]);
 7724        m64[1] = ByteReverseWord64(m64[1]);
 7725    #endif
 7726    }
 7727#endif
 7728
 7729}
 7730
 7731#endif /* GCM_TABLE */
 7732#endif
 7733
 7734#if defined(WOLFSSL_AESNI) && defined(USE_INTEL_SPEEDUP)
 7735    #define HAVE_INTEL_AVX1
 7736    #define HAVE_INTEL_AVX2
 7737#endif
 7738
 7739#if defined(WOLFSSL_AESNI) && defined(GCM_TABLE_4BIT) && \
 7740    defined(WC_C_DYNAMIC_FALLBACK)
 7741void GCM_generate_m0_aesni(const unsigned char *h, unsigned char *m)
 7742                           XASM_LINK("GCM_generate_m0_aesni");
 7743#ifdef HAVE_INTEL_AVX1
 7744void GCM_generate_m0_avx1(const unsigned char *h, unsigned char *m)
 7745                          XASM_LINK("GCM_generate_m0_avx1");
 7746#endif
 7747#ifdef HAVE_INTEL_AVX2
 7748void GCM_generate_m0_avx2(const unsigned char *h, unsigned char *m)
 7749                          XASM_LINK("GCM_generate_m0_avx2");
 7750#endif
 7751#endif /* WOLFSSL_AESNI && GCM_TABLE_4BIT && WC_C_DYNAMIC_FALLBACK */
 7752
 7753/* Software AES - GCM SetKey */
 7754int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len)
 7755{
 7756    int  ret;
 7757    byte iv[WC_AES_BLOCK_SIZE];
 7758
 7759    #ifdef WOLFSSL_IMX6_CAAM_BLOB
 7760        byte   local[32];
 7761        word32 localSz = 32;
 7762
 7763        if (len == (16 + WC_CAAM_BLOB_SZ) ||
 7764          len == (24 + WC_CAAM_BLOB_SZ) ||
 7765          len == (32 + WC_CAAM_BLOB_SZ)) {
 7766            if (wc_caamOpenBlob((byte*)key, len, local, &localSz) != 0) {
 7767                 return BAD_FUNC_ARG;
 7768            }
 7769
 7770            /* set local values */
 7771            key = local;
 7772            len = localSz;
 7773        }
 7774    #endif
 7775
 7776    if (!((len == 16) || (len == 24) || (len == 32)))
 7777        return BAD_FUNC_ARG;
 7778
 7779    if (aes == NULL || key == NULL) {
 7780#ifdef WOLFSSL_IMX6_CAAM_BLOB
 7781        ForceZero(local, sizeof(local));
 7782#endif
 7783        return BAD_FUNC_ARG;
 7784    }
 7785#ifdef OPENSSL_EXTRA
 7786    XMEMSET(aes->gcm.aadH, 0, sizeof(aes->gcm.aadH));
 7787    aes->gcm.aadLen = 0;
 7788#endif
 7789    XMEMSET(iv, 0, WC_AES_BLOCK_SIZE);
 7790    ret = wc_AesSetKey(aes, key, len, iv, AES_ENCRYPTION);
 7791#ifdef WOLF_CRYPTO_CB_ONLY_AES
 7792    /* do key scheduling so that ECB-only devices can still do GCM */
 7793    if (ret == 0) {
 7794        ret = wc_CryptoCb_AesEcbEncrypt(aes, aes->gcm.H, iv, WC_AES_BLOCK_SIZE);
 7795#if defined(GCM_TABLE) || defined(GCM_TABLE_4BIT)
 7796        if (ret == 0)
 7797            GenerateM0(&aes->gcm);
 7798#endif
 7799    }
 7800    return ret;
 7801#endif
 7802#ifdef WOLFSSL_AESGCM_STREAM
 7803    aes->gcmKeySet = 1;
 7804#endif
 7805    #if defined(WOLFSSL_SECO_CAAM)
 7806        if (aes->devId == WOLFSSL_SECO_DEVID) {
 7807            return ret;
 7808        }
 7809    #endif /* WOLFSSL_SECO_CAAM */
 7810
 7811    #if defined(WOLFSSL_RENESAS_FSPSM_CRYPTONLY) && \
 7812        !defined(NO_WOLFSSL_RENESAS_FSPSM_AES)
 7813        return ret;
 7814    #endif /* WOLFSSL_RENESAS_RSIP && WOLFSSL_RENESAS_FSPSM_CRYPTONLY*/
 7815
 7816/* GCM setup needs one AES block encrypt of the all-zero IV to generate
 7817 * the hash subkey H. STM32_CRYPTO stores only the raw key (no expanded
 7818 * key schedule), so the ARMASM AES_ECB_encrypt helpers used here cannot
 7819 * be used. Excluding STM32_CRYPTO from this block falls back to the
 7820 * non-ARMASM wc_AesEncrypt implementation, which on STM32 routes to
 7821 * CRYP. */
 7822#if defined(WOLFSSL_ARMASM) && !defined(STM32_CRYPTO)
 7823    if (ret == 0) {
 7824#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
 7825    #if !defined(__aarch64__)
 7826        AES_GCM_set_key_AARCH32(iv, (byte*)aes->key, aes->gcm.H, aes->rounds);
 7827    #else
 7828        if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) {
 7829            AES_GCM_set_key_AARCH64(iv, (byte*)aes->key, aes->gcm.H,
 7830                aes->rounds);
 7831        }
 7832        else
 7833    #endif /* !__aarch64__ */
 7834#endif /* !WOLFSSL_ARMASM_NO_HW_CRYPTO */
 7835#if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON) && \
 7836    defined(WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP)
 7837        {
 7838            AES_ECB_encrypt_NEON(iv, aes->gcm.H, WC_AES_BLOCK_SIZE,
 7839                (const unsigned char*)aes->key, aes->rounds);
 7840        }
 7841#elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
 7842        {
 7843            AES_ECB_encrypt(iv, aes->gcm.H, WC_AES_BLOCK_SIZE,
 7844                (const unsigned char*)aes->key, aes->rounds);
 7845        #if defined(GCM_TABLE) || defined(GCM_TABLE_4BIT)
 7846            GenerateM0(&aes->gcm);
 7847        #endif /* GCM_TABLE */
 7848        }
 7849#endif
 7850    }
 7851#else
 7852#if !defined(FREESCALE_LTC_AES_GCM) && !defined(WOLFSSL_PSOC6_CRYPTO)
 7853
 7854
 7855#ifdef WOLF_CRYPTO_CB_AES_SETKEY
 7856    if ((ret == 0) && (aes->devId != INVALID_DEVID && aes->devCtx != NULL)) {
 7857        /* SE owns key - skip H and M table generation */
 7858    }
 7859    else
 7860#endif
 7861    if (ret == 0) {
 7862        VECTOR_REGISTERS_PUSH;
 7863
 7864        /* Generate H = AES_Encrypt(key, 0^128) */
 7865        ret = wc_AesEncrypt(aes, iv, aes->gcm.H);
 7866
 7867        if (ret == 0) {
 7868#if defined(GCM_TABLE) || defined(GCM_TABLE_4BIT)
 7869    #if defined(WOLFSSL_AESNI) && defined(GCM_TABLE_4BIT)
 7870            if (aes->use_aesni) {
 7871        #if defined(WC_C_DYNAMIC_FALLBACK)
 7872            #ifdef HAVE_INTEL_AVX2
 7873                if (IS_INTEL_AVX2(intel_flags)) {
 7874                    GCM_generate_m0_avx2(aes->gcm.H,
 7875                        (byte*)aes->gcm.M0);
 7876                }
 7877                else
 7878            #endif
 7879            #if defined(HAVE_INTEL_AVX1)
 7880                if (IS_INTEL_AVX1(intel_flags)) {
 7881                    GCM_generate_m0_avx1(aes->gcm.H,
 7882                        (byte*)aes->gcm.M0);
 7883                }
 7884                else
 7885            #endif
 7886                {
 7887                    GCM_generate_m0_aesni(aes->gcm.H,
 7888                        (byte*)aes->gcm.M0);
 7889                }
 7890        #endif /* WC_C_DYNAMIC_FALLBACK */
 7891            }
 7892            else
 7893    #endif /* AESNI */
 7894            {
 7895                GenerateM0(&aes->gcm);
 7896            }
 7897#endif /* GCM_TABLE || GCM_TABLE_4BIT */
 7898        }
 7899
 7900        VECTOR_REGISTERS_POP;
 7901    }
 7902#endif /* !FREESCALE_LTC_AES_GCM && !WOLFSSL_PSOC6_CRYPTO */
 7903#endif
 7904
 7905#if defined(WOLFSSL_XILINX_CRYPT) || defined(WOLFSSL_AFALG_XILINX_AES)
 7906    wc_AesGcmSetKey_ex(aes, key, len, WOLFSSL_XILINX_AES_KEY_SRC);
 7907#endif
 7908
 7909#ifdef WOLF_CRYPTO_CB
 7910    if (aes->devId != INVALID_DEVID) {
 7911    #ifdef WOLF_CRYPTO_CB_AES_SETKEY
 7912        if (aes->devCtx != NULL) {
 7913            /* SE owns key - don't copy to devKey */
 7914        }
 7915        else
 7916    #endif
 7917        {
 7918            XMEMCPY(aes->devKey, key, len);
 7919        }
 7920    }
 7921#endif
 7922
 7923#ifdef WOLFSSL_IMX6_CAAM_BLOB
 7924    ForceZero(local, sizeof(local));
 7925#endif
 7926    return ret;
 7927}
 7928
 7929
 7930#ifdef WOLFSSL_AESNI
 7931
 7932void AES_GCM_encrypt_aesni(const unsigned char *in, unsigned char *out,
 7933                     const unsigned char* addt, const unsigned char* ivec,
 7934                     unsigned char *tag, word32 nbytes,
 7935                     word32 abytes, word32 ibytes,
 7936                     word32 tbytes, const unsigned char* key, int nr)
 7937                     XASM_LINK("AES_GCM_encrypt_aesni");
 7938#ifdef HAVE_INTEL_AVX1
 7939void AES_GCM_encrypt_avx1(const unsigned char *in, unsigned char *out,
 7940                          const unsigned char* addt, const unsigned char* ivec,
 7941                          unsigned char *tag, word32 nbytes,
 7942                          word32 abytes, word32 ibytes,
 7943                          word32 tbytes, const unsigned char* key,
 7944                          int nr)
 7945                          XASM_LINK("AES_GCM_encrypt_avx1");
 7946#ifdef HAVE_INTEL_AVX2
 7947void AES_GCM_encrypt_avx2(const unsigned char *in, unsigned char *out,
 7948                          const unsigned char* addt, const unsigned char* ivec,
 7949                          unsigned char *tag, word32 nbytes,
 7950                          word32 abytes, word32 ibytes,
 7951                          word32 tbytes, const unsigned char* key,
 7952                          int nr)
 7953                          XASM_LINK("AES_GCM_encrypt_avx2");
 7954#endif /* HAVE_INTEL_AVX2 */
 7955#endif /* HAVE_INTEL_AVX1 */
 7956
 7957#ifdef HAVE_AES_DECRYPT
 7958void AES_GCM_decrypt_aesni(const unsigned char *in, unsigned char *out,
 7959                     const unsigned char* addt, const unsigned char* ivec,
 7960                     const unsigned char *tag, word32 nbytes, word32 abytes,
 7961                     word32 ibytes, word32 tbytes, const unsigned char* key,
 7962                     int nr, int* res)
 7963                     XASM_LINK("AES_GCM_decrypt_aesni");
 7964#ifdef HAVE_INTEL_AVX1
 7965void AES_GCM_decrypt_avx1(const unsigned char *in, unsigned char *out,
 7966                          const unsigned char* addt, const unsigned char* ivec,
 7967                          const unsigned char *tag, word32 nbytes,
 7968                          word32 abytes, word32 ibytes, word32 tbytes,
 7969                          const unsigned char* key, int nr, int* res)
 7970                          XASM_LINK("AES_GCM_decrypt_avx1");
 7971#ifdef HAVE_INTEL_AVX2
 7972void AES_GCM_decrypt_avx2(const unsigned char *in, unsigned char *out,
 7973                          const unsigned char* addt, const unsigned char* ivec,
 7974                          const unsigned char *tag, word32 nbytes,
 7975                          word32 abytes, word32 ibytes, word32 tbytes,
 7976                          const unsigned char* key, int nr, int* res)
 7977                          XASM_LINK("AES_GCM_decrypt_avx2");
 7978#endif /* HAVE_INTEL_AVX2 */
 7979#endif /* HAVE_INTEL_AVX1 */
 7980#endif /* HAVE_AES_DECRYPT */
 7981
 7982#endif /* WOLFSSL_AESNI */
 7983
 7984#if !defined(WOLFSSL_ARMASM) || defined(__aarch64__) || \
 7985    defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
 7986#if defined(GCM_SMALL)
 7987static void GMULT(byte* X, byte* Y)
 7988{
 7989    byte Z[WC_AES_BLOCK_SIZE];
 7990    byte V[WC_AES_BLOCK_SIZE];
 7991    int i, j;
 7992
 7993    XMEMSET(Z, 0, WC_AES_BLOCK_SIZE);
 7994    XMEMCPY(V, X, WC_AES_BLOCK_SIZE);
 7995    for (i = 0; i < WC_AES_BLOCK_SIZE; i++)
 7996    {
 7997        byte y = Y[i];
 7998        for (j = 0; j < 8; j++)
 7999        {
 8000            if (y & 0x80) {
 8001                xorbuf(Z, V, WC_AES_BLOCK_SIZE);
 8002            }
 8003
 8004            RIGHTSHIFTX(V);
 8005            y = y << 1;
 8006        }
 8007    }
 8008    XMEMCPY(X, Z, WC_AES_BLOCK_SIZE);
 8009}
 8010
 8011
 8012void GHASH(Gcm* gcm, const byte* a, word32 aSz, const byte* c,
 8013    word32 cSz, byte* s, word32 sSz)
 8014{
 8015    byte x[WC_AES_BLOCK_SIZE];
 8016    byte scratch[WC_AES_BLOCK_SIZE];
 8017    word32 blocks, partial;
 8018    byte* h;
 8019
 8020    if (gcm == NULL) {
 8021        return;
 8022    }
 8023
 8024    h = gcm->H;
 8025    XMEMSET(x, 0, WC_AES_BLOCK_SIZE);
 8026
 8027    /* Hash in A, the Additional Authentication Data */
 8028    if (aSz != 0 && a != NULL) {
 8029        blocks = aSz / WC_AES_BLOCK_SIZE;
 8030        partial = aSz % WC_AES_BLOCK_SIZE;
 8031        while (blocks--) {
 8032            xorbuf(x, a, WC_AES_BLOCK_SIZE);
 8033            GMULT(x, h);
 8034            a += WC_AES_BLOCK_SIZE;
 8035        }
 8036        if (partial != 0) {
 8037            XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
 8038            XMEMCPY(scratch, a, partial);
 8039            xorbuf(x, scratch, WC_AES_BLOCK_SIZE);
 8040            GMULT(x, h);
 8041        }
 8042    }
 8043
 8044    /* Hash in C, the Ciphertext */
 8045    if (cSz != 0 && c != NULL) {
 8046        blocks = cSz / WC_AES_BLOCK_SIZE;
 8047        partial = cSz % WC_AES_BLOCK_SIZE;
 8048        while (blocks--) {
 8049            xorbuf(x, c, WC_AES_BLOCK_SIZE);
 8050            GMULT(x, h);
 8051            c += WC_AES_BLOCK_SIZE;
 8052        }
 8053        if (partial != 0) {
 8054            XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
 8055            XMEMCPY(scratch, c, partial);
 8056            xorbuf(x, scratch, WC_AES_BLOCK_SIZE);
 8057            GMULT(x, h);
 8058        }
 8059    }
 8060
 8061    /* Hash in the lengths of A and C in bits */
 8062    FlattenSzInBits(&scratch[0], aSz);
 8063    FlattenSzInBits(&scratch[8], cSz);
 8064    xorbuf(x, scratch, WC_AES_BLOCK_SIZE);
 8065    GMULT(x, h);
 8066
 8067    /* Copy the result into s. */
 8068    XMEMCPY(s, x, sSz);
 8069}
 8070
 8071#ifdef WOLFSSL_AESGCM_STREAM
 8072/* No extra initialization for small implementation.
 8073 *
 8074 * @param [in] aes  AES GCM object.
 8075 */
 8076#define GHASH_INIT_EXTRA(aes) WC_DO_NOTHING
 8077
 8078/* GHASH one block of data..
 8079 *
 8080 * XOR block into tag and GMULT with H.
 8081 *
 8082 * @param [in, out] aes    AES GCM object.
 8083 * @param [in]      block  Block of AAD or cipher text.
 8084 */
 8085#define GHASH_ONE_BLOCK_SW(aes, block)                  \
 8086    do {                                                \
 8087        xorbuf(AES_TAG(aes), block, WC_AES_BLOCK_SIZE); \
 8088        GMULT(AES_TAG(aes), (aes)->gcm.H);              \
 8089    }                                                   \
 8090    while (0)
 8091#endif /* WOLFSSL_AESGCM_STREAM */
 8092
 8093#if defined(WOLFSSL_ARMASM) && (!defined(__aarch64__) || \
 8094    defined(WOLFSSL_ARMASM_NO_NEON))
 8095static void GCM_gmult_len_armasm_C(
 8096    byte* x, const byte* h, const unsigned char* a, unsigned long len)
 8097{
 8098    byte Z[AES_BLOCK_SIZE];
 8099    byte V[AES_BLOCK_SIZE];
 8100    int i;
 8101    int j;
 8102
 8103    while (len >= AES_BLOCK_SIZE) {
 8104        xorbuf(x, a, AES_BLOCK_SIZE);
 8105        XMEMSET(Z, 0, AES_BLOCK_SIZE);
 8106        XMEMCPY(V, x, AES_BLOCK_SIZE);
 8107        for (i = 0; i < AES_BLOCK_SIZE; i++) {
 8108            byte y = h[i];
 8109            for (j = 0; j < 8; j++) {
 8110                if (y & 0x80) {
 8111                    xorbuf(Z, V, AES_BLOCK_SIZE);
 8112                }
 8113                RIGHTSHIFTX(V);
 8114                y = y << 1;
 8115            }
 8116        }
 8117        XMEMCPY(x, Z, AES_BLOCK_SIZE);
 8118        len -= AES_BLOCK_SIZE;
 8119        a += AES_BLOCK_SIZE;
 8120    }
 8121}
 8122
 8123#define GCM_GMULT_LEN(gcm, x, a, len) \
 8124    GCM_gmult_len_armasm_C(x, (gcm)->H, a, len)
 8125#elif defined(WOLFSSL_ARMASM)
 8126#define GCM_GMULT_LEN(gcm, x, a, len) \
 8127    GCM_gmult_len_NEON(x, (const byte*)((gcm)->H), a, len)
 8128#endif
 8129
 8130#elif defined(GCM_TABLE)
 8131
 8132#if defined(WOLFSSL_ARMASM) && (defined(__aarch64__) || \
 8133    defined(WOLFSSL_ARMASM_NO_HW_CRYPTO))
 8134#if !defined(WOLFSSL_ARMASM_NO_NEON) && defined(__aarch64__)
 8135#define GCM_GMULT_LEN(gcm, x, a, len) \
 8136    GCM_gmult_len_NEON(x, (const byte*)((gcm)->H), a, len)
 8137#else
 8138#define GCM_GMULT_LEN(gcm, x, a, len) \
 8139    GCM_gmult_len(x, (const byte**)((gcm)->M0), a, len)
 8140#endif
 8141#else
 8142ALIGN16 static const byte R[256][2] = {
 8143    {0x00, 0x00}, {0x01, 0xc2}, {0x03, 0x84}, {0x02, 0x46},
 8144    {0x07, 0x08}, {0x06, 0xca}, {0x04, 0x8c}, {0x05, 0x4e},
 8145    {0x0e, 0x10}, {0x0f, 0xd2}, {0x0d, 0x94}, {0x0c, 0x56},
 8146    {0x09, 0x18}, {0x08, 0xda}, {0x0a, 0x9c}, {0x0b, 0x5e},
 8147    {0x1c, 0x20}, {0x1d, 0xe2}, {0x1f, 0xa4}, {0x1e, 0x66},
 8148    {0x1b, 0x28}, {0x1a, 0xea}, {0x18, 0xac}, {0x19, 0x6e},
 8149    {0x12, 0x30}, {0x13, 0xf2}, {0x11, 0xb4}, {0x10, 0x76},
 8150    {0x15, 0x38}, {0x14, 0xfa}, {0x16, 0xbc}, {0x17, 0x7e},
 8151    {0x38, 0x40}, {0x39, 0x82}, {0x3b, 0xc4}, {0x3a, 0x06},
 8152    {0x3f, 0x48}, {0x3e, 0x8a}, {0x3c, 0xcc}, {0x3d, 0x0e},
 8153    {0x36, 0x50}, {0x37, 0x92}, {0x35, 0xd4}, {0x34, 0x16},
 8154    {0x31, 0x58}, {0x30, 0x9a}, {0x32, 0xdc}, {0x33, 0x1e},
 8155    {0x24, 0x60}, {0x25, 0xa2}, {0x27, 0xe4}, {0x26, 0x26},
 8156    {0x23, 0x68}, {0x22, 0xaa}, {0x20, 0xec}, {0x21, 0x2e},
 8157    {0x2a, 0x70}, {0x2b, 0xb2}, {0x29, 0xf4}, {0x28, 0x36},
 8158    {0x2d, 0x78}, {0x2c, 0xba}, {0x2e, 0xfc}, {0x2f, 0x3e},
 8159    {0x70, 0x80}, {0x71, 0x42}, {0x73, 0x04}, {0x72, 0xc6},
 8160    {0x77, 0x88}, {0x76, 0x4a}, {0x74, 0x0c}, {0x75, 0xce},
 8161    {0x7e, 0x90}, {0x7f, 0x52}, {0x7d, 0x14}, {0x7c, 0xd6},
 8162    {0x79, 0x98}, {0x78, 0x5a}, {0x7a, 0x1c}, {0x7b, 0xde},
 8163    {0x6c, 0xa0}, {0x6d, 0x62}, {0x6f, 0x24}, {0x6e, 0xe6},
 8164    {0x6b, 0xa8}, {0x6a, 0x6a}, {0x68, 0x2c}, {0x69, 0xee},
 8165    {0x62, 0xb0}, {0x63, 0x72}, {0x61, 0x34}, {0x60, 0xf6},
 8166    {0x65, 0xb8}, {0x64, 0x7a}, {0x66, 0x3c}, {0x67, 0xfe},
 8167    {0x48, 0xc0}, {0x49, 0x02}, {0x4b, 0x44}, {0x4a, 0x86},
 8168    {0x4f, 0xc8}, {0x4e, 0x0a}, {0x4c, 0x4c}, {0x4d, 0x8e},
 8169    {0x46, 0xd0}, {0x47, 0x12}, {0x45, 0x54}, {0x44, 0x96},
 8170    {0x41, 0xd8}, {0x40, 0x1a}, {0x42, 0x5c}, {0x43, 0x9e},
 8171    {0x54, 0xe0}, {0x55, 0x22}, {0x57, 0x64}, {0x56, 0xa6},
 8172    {0x53, 0xe8}, {0x52, 0x2a}, {0x50, 0x6c}, {0x51, 0xae},
 8173    {0x5a, 0xf0}, {0x5b, 0x32}, {0x59, 0x74}, {0x58, 0xb6},
 8174    {0x5d, 0xf8}, {0x5c, 0x3a}, {0x5e, 0x7c}, {0x5f, 0xbe},
 8175    {0xe1, 0x00}, {0xe0, 0xc2}, {0xe2, 0x84}, {0xe3, 0x46},
 8176    {0xe6, 0x08}, {0xe7, 0xca}, {0xe5, 0x8c}, {0xe4, 0x4e},
 8177    {0xef, 0x10}, {0xee, 0xd2}, {0xec, 0x94}, {0xed, 0x56},
 8178    {0xe8, 0x18}, {0xe9, 0xda}, {0xeb, 0x9c}, {0xea, 0x5e},
 8179    {0xfd, 0x20}, {0xfc, 0xe2}, {0xfe, 0xa4}, {0xff, 0x66},
 8180    {0xfa, 0x28}, {0xfb, 0xea}, {0xf9, 0xac}, {0xf8, 0x6e},
 8181    {0xf3, 0x30}, {0xf2, 0xf2}, {0xf0, 0xb4}, {0xf1, 0x76},
 8182    {0xf4, 0x38}, {0xf5, 0xfa}, {0xf7, 0xbc}, {0xf6, 0x7e},
 8183    {0xd9, 0x40}, {0xd8, 0x82}, {0xda, 0xc4}, {0xdb, 0x06},
 8184    {0xde, 0x48}, {0xdf, 0x8a}, {0xdd, 0xcc}, {0xdc, 0x0e},
 8185    {0xd7, 0x50}, {0xd6, 0x92}, {0xd4, 0xd4}, {0xd5, 0x16},
 8186    {0xd0, 0x58}, {0xd1, 0x9a}, {0xd3, 0xdc}, {0xd2, 0x1e},
 8187    {0xc5, 0x60}, {0xc4, 0xa2}, {0xc6, 0xe4}, {0xc7, 0x26},
 8188    {0xc2, 0x68}, {0xc3, 0xaa}, {0xc1, 0xec}, {0xc0, 0x2e},
 8189    {0xcb, 0x70}, {0xca, 0xb2}, {0xc8, 0xf4}, {0xc9, 0x36},
 8190    {0xcc, 0x78}, {0xcd, 0xba}, {0xcf, 0xfc}, {0xce, 0x3e},
 8191    {0x91, 0x80}, {0x90, 0x42}, {0x92, 0x04}, {0x93, 0xc6},
 8192    {0x96, 0x88}, {0x97, 0x4a}, {0x95, 0x0c}, {0x94, 0xce},
 8193    {0x9f, 0x90}, {0x9e, 0x52}, {0x9c, 0x14}, {0x9d, 0xd6},
 8194    {0x98, 0x98}, {0x99, 0x5a}, {0x9b, 0x1c}, {0x9a, 0xde},
 8195    {0x8d, 0xa0}, {0x8c, 0x62}, {0x8e, 0x24}, {0x8f, 0xe6},
 8196    {0x8a, 0xa8}, {0x8b, 0x6a}, {0x89, 0x2c}, {0x88, 0xee},
 8197    {0x83, 0xb0}, {0x82, 0x72}, {0x80, 0x34}, {0x81, 0xf6},
 8198    {0x84, 0xb8}, {0x85, 0x7a}, {0x87, 0x3c}, {0x86, 0xfe},
 8199    {0xa9, 0xc0}, {0xa8, 0x02}, {0xaa, 0x44}, {0xab, 0x86},
 8200    {0xae, 0xc8}, {0xaf, 0x0a}, {0xad, 0x4c}, {0xac, 0x8e},
 8201    {0xa7, 0xd0}, {0xa6, 0x12}, {0xa4, 0x54}, {0xa5, 0x96},
 8202    {0xa0, 0xd8}, {0xa1, 0x1a}, {0xa3, 0x5c}, {0xa2, 0x9e},
 8203    {0xb5, 0xe0}, {0xb4, 0x22}, {0xb6, 0x64}, {0xb7, 0xa6},
 8204    {0xb2, 0xe8}, {0xb3, 0x2a}, {0xb1, 0x6c}, {0xb0, 0xae},
 8205    {0xbb, 0xf0}, {0xba, 0x32}, {0xb8, 0x74}, {0xb9, 0xb6},
 8206    {0xbc, 0xf8}, {0xbd, 0x3a}, {0xbf, 0x7c}, {0xbe, 0xbe} };
 8207
 8208
 8209static void GMULT(byte *x, byte m[256][WC_AES_BLOCK_SIZE])
 8210{
 8211#if !defined(WORD64_AVAILABLE) || defined(BIG_ENDIAN_ORDER)
 8212    int i, j;
 8213    byte Z[WC_AES_BLOCK_SIZE];
 8214    byte a;
 8215
 8216    XMEMSET(Z, 0, sizeof(Z));
 8217
 8218    for (i = 15; i > 0; i--) {
 8219        xorbuf(Z, m[x[i]], WC_AES_BLOCK_SIZE);
 8220        a = Z[15];
 8221
 8222        for (j = 15; j > 0; j--) {
 8223            Z[j] = Z[j-1];
 8224        }
 8225
 8226        Z[0]  = R[a][0];
 8227        Z[1] ^= R[a][1];
 8228    }
 8229    xorbuf(Z, m[x[0]], WC_AES_BLOCK_SIZE);
 8230
 8231    XMEMCPY(x, Z, WC_AES_BLOCK_SIZE);
 8232#elif defined(WC_32BIT_CPU)
 8233    byte Z[WC_AES_BLOCK_SIZE + WC_AES_BLOCK_SIZE];
 8234    byte a;
 8235    word32* pZ;
 8236    word32* pm;
 8237    word32* px = (word32*)(x);
 8238    int i;
 8239
 8240    pZ = (word32*)(Z + 15 + 1);
 8241    pm = (word32*)(m[x[15]]);
 8242    pZ[0] = pm[0];
 8243    pZ[1] = pm[1];
 8244    pZ[2] = pm[2];
 8245    pZ[3] = pm[3];
 8246    a = Z[16 + 15];
 8247    Z[15]  = R[a][0];
 8248    Z[16] ^= R[a][1];
 8249    for (i = 14; i > 0; i--) {
 8250        pZ = (word32*)(Z + i + 1);
 8251        pm = (word32*)(m[x[i]]);
 8252        pZ[0] ^= pm[0];
 8253        pZ[1] ^= pm[1];
 8254        pZ[2] ^= pm[2];
 8255        pZ[3] ^= pm[3];
 8256        a = Z[16 + i];
 8257        Z[i]    = R[a][0];
 8258        Z[i+1] ^= R[a][1];
 8259    }
 8260    pZ = (word32*)(Z + 1);
 8261    pm = (word32*)(m[x[0]]);
 8262    px[0] = pZ[0] ^ pm[0]; px[1] = pZ[1] ^ pm[1];
 8263    px[2] = pZ[2] ^ pm[2]; px[3] = pZ[3] ^ pm[3];
 8264#else
 8265    byte Z[WC_AES_BLOCK_SIZE + WC_AES_BLOCK_SIZE];
 8266    byte a;
 8267    word64* pZ;
 8268    word64* pm;
 8269    word64* px = (word64*)(x);
 8270    int i;
 8271
 8272    pZ = (word64*)(Z + 15 + 1);
 8273    pm = (word64*)(m[x[15]]);
 8274    pZ[0] = pm[0];
 8275    pZ[1] = pm[1];
 8276    a = Z[16 + 15];
 8277    Z[15]  = R[a][0];
 8278    Z[16] ^= R[a][1];
 8279    for (i = 14; i > 0; i--) {
 8280        pZ = (word64*)(Z + i + 1);
 8281        pm = (word64*)(m[x[i]]);
 8282        pZ[0] ^= pm[0];
 8283        pZ[1] ^= pm[1];
 8284        a = Z[16 + i];
 8285        Z[i]    = R[a][0];
 8286        Z[i+1] ^= R[a][1];
 8287    }
 8288    pZ = (word64*)(Z + 1);
 8289    pm = (word64*)(m[x[0]]);
 8290    px[0] = pZ[0] ^ pm[0]; px[1] = pZ[1] ^ pm[1];
 8291#endif
 8292}
 8293#endif
 8294
 8295void GHASH(Gcm* gcm, const byte* a, word32 aSz, const byte* c,
 8296    word32 cSz, byte* s, word32 sSz)
 8297{
 8298    byte x[WC_AES_BLOCK_SIZE];
 8299    byte scratch[WC_AES_BLOCK_SIZE];
 8300    word32 blocks, partial;
 8301
 8302    if (gcm == NULL) {
 8303        return;
 8304    }
 8305
 8306    XMEMSET(x, 0, WC_AES_BLOCK_SIZE);
 8307
 8308    /* Hash in A, the Additional Authentication Data */
 8309    if (aSz != 0 && a != NULL) {
 8310        blocks = aSz / WC_AES_BLOCK_SIZE;
 8311        partial = aSz % WC_AES_BLOCK_SIZE;
 8312    #ifdef GCM_GMULT_LEN
 8313        if (blocks > 0) {
 8314            GCM_GMULT_LEN(gcm, x, a, blocks * WC_AES_BLOCK_SIZE);
 8315            a += blocks * WC_AES_BLOCK_SIZE;
 8316        }
 8317        if (partial != 0) {
 8318            XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
 8319            XMEMCPY(scratch, a, partial);
 8320            GCM_GMULT_LEN(gcm, x, scratch, WC_AES_BLOCK_SIZE);
 8321        }
 8322    #else
 8323        while (blocks--) {
 8324            xorbuf(x, a, WC_AES_BLOCK_SIZE);
 8325            GMULT(x, gcm->M0);
 8326            a += WC_AES_BLOCK_SIZE;
 8327        }
 8328        if (partial != 0) {
 8329            XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
 8330            XMEMCPY(scratch, a, partial);
 8331            xorbuf(x, scratch, WC_AES_BLOCK_SIZE);
 8332            GMULT(x, gcm->M0);
 8333        }
 8334    #endif
 8335    }
 8336
 8337    /* Hash in C, the Ciphertext */
 8338    if (cSz != 0 && c != NULL) {
 8339        blocks = cSz / WC_AES_BLOCK_SIZE;
 8340        partial = cSz % WC_AES_BLOCK_SIZE;
 8341    #ifdef GCM_GMULT_LEN
 8342        if (blocks > 0) {
 8343            GCM_GMULT_LEN(gcm, x, c, blocks * WC_AES_BLOCK_SIZE);
 8344            c += blocks * WC_AES_BLOCK_SIZE;
 8345        }
 8346        if (partial != 0) {
 8347            XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
 8348            XMEMCPY(scratch, c, partial);
 8349            GCM_GMULT_LEN(gcm, x, scratch, WC_AES_BLOCK_SIZE);
 8350        }
 8351    #else
 8352        while (blocks--) {
 8353            xorbuf(x, c, WC_AES_BLOCK_SIZE);
 8354            GMULT(x, gcm->M0);
 8355            c += WC_AES_BLOCK_SIZE;
 8356        }
 8357        if (partial != 0) {
 8358            XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
 8359            XMEMCPY(scratch, c, partial);
 8360            xorbuf(x, scratch, WC_AES_BLOCK_SIZE);
 8361            GMULT(x, gcm->M0);
 8362        }
 8363    #endif
 8364    }
 8365
 8366    /* Hash in the lengths of A and C in bits */
 8367    FlattenSzInBits(&scratch[0], aSz);
 8368    FlattenSzInBits(&scratch[8], cSz);
 8369#ifdef GCM_GMULT_LEN
 8370    GCM_GMULT_LEN(gcm, x, scratch, WC_AES_BLOCK_SIZE);
 8371#else
 8372    xorbuf(x, scratch, WC_AES_BLOCK_SIZE);
 8373    GMULT(x, gcm->M0);
 8374#endif
 8375
 8376    /* Copy the result into s. */
 8377    XMEMCPY(s, x, sSz);
 8378}
 8379
 8380#ifdef WOLFSSL_AESGCM_STREAM
 8381/* No extra initialization for table implementation.
 8382 *
 8383 * @param [in] aes  AES GCM object.
 8384 */
 8385#define GHASH_INIT_EXTRA(aes) WC_DO_NOTHING
 8386
 8387/* GHASH one block of data..
 8388 *
 8389 * XOR block into tag and GMULT with H using pre-computed table.
 8390 *
 8391 * @param [in, out] aes    AES GCM object.
 8392 * @param [in]      block  Block of AAD or cipher text.
 8393 */
 8394#define GHASH_ONE_BLOCK_SW(aes, block)                  \
 8395    do {                                                \
 8396        xorbuf(AES_TAG(aes), block, WC_AES_BLOCK_SIZE); \
 8397        GMULT(AES_TAG(aes), aes->gcm.M0);               \
 8398    }                                                   \
 8399    while (0)
 8400#endif /* WOLFSSL_AESGCM_STREAM */
 8401/* end GCM_TABLE */
 8402#elif defined(GCM_TABLE_4BIT)
 8403
 8404#if defined(WOLFSSL_ARMASM) && (defined(__aarch64__) || \
 8405    defined(WOLFSSL_ARMASM_NO_HW_CRYPTO))
 8406#if !defined(WOLFSSL_ARMASM_NO_NEON) && defined(__aarch64__)
 8407#define GCM_GMULT_LEN(gcm, x, a, len) \
 8408    GCM_gmult_len_NEON(x, (const byte*)((gcm)->H), a, len)
 8409#define GMULT(x, m)                                                      \
 8410    GCM_gmult_NEON(x, (const byte**)m)
 8411#else
 8412#define GCM_GMULT_LEN(gcm, x, a, len) \
 8413    GCM_gmult_len(x, (const byte**)((gcm)->M0), a, len)
 8414#define GMULT(x, m)                                                      \
 8415    GCM_gmult(x, (const byte**)m)
 8416#endif
 8417#else
 8418/* remainder = x^7 + x^2 + x^1 + 1 => 0xe1
 8419 *  R shifts right a reverse bit pair of bytes such that:
 8420 *     R(b0, b1) => b1 = (b1 >> 1) | (b0 << 7); b0 >>= 1
 8421 *  0 => 0, 0, 0, 0 => R(R(R(00,00) ^ 00,00) ^ 00,00) ^ 00,00 = 00,00
 8422 *  8 => 0, 0, 0, 1 => R(R(R(00,00) ^ 00,00) ^ 00,00) ^ e1,00 = e1,00
 8423 *  4 => 0, 0, 1, 0 => R(R(R(00,00) ^ 00,00) ^ e1,00) ^ 00,00 = 70,80
 8424 *  2 => 0, 1, 0, 0 => R(R(R(00,00) ^ e1,00) ^ 00,00) ^ 00,00 = 38,40
 8425 *  1 => 1, 0, 0, 0 => R(R(R(e1,00) ^ 00,00) ^ 00,00) ^ 00,00 = 1c,20
 8426 *  To calculate te rest, XOR result for each bit.
 8427 *   e.g. 6 = 4 ^ 2 => 48,c0
 8428 *
 8429 * Second half is same values rotated by 4-bits.
 8430 */
 8431#if defined(WC_16BIT_CPU)
 8432static const byte R[16][2] = {
 8433    {0x00, 0x00}, {0x1c, 0x20}, {0x38, 0x40}, {0x24, 0x60},
 8434    {0x70, 0x80}, {0x6c, 0xa0}, {0x48, 0xc0}, {0x54, 0xe0},
 8435    {0xe1, 0x00}, {0xfd, 0x20}, {0xd9, 0x40}, {0xc5, 0x60},
 8436    {0x91, 0x80}, {0x8d, 0xa0}, {0xa9, 0xc0}, {0xb5, 0xe0},
 8437};
 8438#elif defined(BIG_ENDIAN_ORDER)
 8439static const word16 R[32] = {
 8440          0x0000,       0x1c20,       0x3840,       0x2460,
 8441          0x7080,       0x6ca0,       0x48c0,       0x54e0,
 8442          0xe100,       0xfd20,       0xd940,       0xc560,
 8443          0x9180,       0x8da0,       0xa9c0,       0xb5e0,
 8444
 8445          0x0000,       0x01c2,       0x0384,       0x0246,
 8446          0x0708,       0x06ca,       0x048c,       0x054e,
 8447          0x0e10,       0x0fd2,       0x0d94,       0x0c56,
 8448          0x0918,       0x08da,       0x0a9c,       0x0b5e,
 8449};
 8450#else
 8451static const word16 R[32] = {
 8452          0x0000,       0x201c,       0x4038,       0x6024,
 8453          0x8070,       0xa06c,       0xc048,       0xe054,
 8454          0x00e1,       0x20fd,       0x40d9,       0x60c5,
 8455          0x8091,       0xa08d,       0xc0a9,       0xe0b5,
 8456
 8457          0x0000,       0xc201,       0x8403,       0x4602,
 8458          0x0807,       0xca06,       0x8c04,       0x4e05,
 8459          0x100e,       0xd20f,       0x940d,       0x560c,
 8460          0x1809,       0xda08,       0x9c0a,       0x5e0b,
 8461};
 8462#endif
 8463
 8464/* Multiply in GF(2^128) defined by polynomial:
 8465 *   x^128 + x^7 + x^2 + x^1 + 1.
 8466 *
 8467 * H: hash key = encrypt(key, 0)
 8468 * x = x * H in field
 8469 *
 8470 * x: cumulative result
 8471 * m: 4-bit table
 8472 *    [0..15] * H
 8473 */
 8474#if defined(WC_16BIT_CPU)
 8475static void GMULT(byte *x, byte m[16][WC_AES_BLOCK_SIZE])
 8476{
 8477    int i, j, n;
 8478    byte Z[WC_AES_BLOCK_SIZE];
 8479    byte a;
 8480
 8481    XMEMSET(Z, 0, sizeof(Z));
 8482
 8483    for (i = 15; i >= 0; i--) {
 8484        for (n = 0; n < 2; n++) {
 8485            if (n == 0)
 8486                xorbuf(Z, m[x[i] & 0xf], WC_AES_BLOCK_SIZE);
 8487            else {
 8488                xorbuf(Z, m[x[i] >> 4], WC_AES_BLOCK_SIZE);
 8489                if (i == 0)
 8490                    break;
 8491            }
 8492            a = Z[15] & 0xf;
 8493
 8494            for (j = 15; j > 0; j--)
 8495                Z[j] = (Z[j-1] << 4) | (Z[j] >> 4);
 8496            Z[0] >>= 4;
 8497
 8498            Z[0] ^= R[a][0];
 8499            Z[1] ^= R[a][1];
 8500        }
 8501    }
 8502
 8503    XMEMCPY(x, Z, WC_AES_BLOCK_SIZE);
 8504}
 8505#elif defined(WC_32BIT_CPU) && defined(BIG_ENDIAN_ORDER)
 8506static WC_INLINE void GMULT(byte *x, byte m[32][WC_AES_BLOCK_SIZE])
 8507{
 8508    int i;
 8509    word32 z8[4] = {0, 0, 0, 0};
 8510    byte a;
 8511    word32* x8 = (word32*)x;
 8512    word32* m8;
 8513    byte xi;
 8514
 8515    for (i = 15; i > 0; i--) {
 8516        xi = x[i];
 8517
 8518        /* XOR in (msn * H) */
 8519        m8 = (word32*)m[xi & 0xf];
 8520        z8[0] ^= m8[0]; z8[1] ^= m8[1]; z8[2] ^= m8[2]; z8[3] ^= m8[3];
 8521
 8522        /* Cache top byte for remainder calculations - lost in rotate. */
 8523        a = (byte)(z8[3] & 0xff);
 8524
 8525        /* Rotate Z by 8-bits */
 8526        z8[3] = (z8[2] << 24) | (z8[3] >> 8);
 8527        z8[2] = (z8[1] << 24) | (z8[2] >> 8);
 8528        z8[1] = (z8[0] << 24) | (z8[1] >> 8);
 8529        z8[0] >>= 8;
 8530
 8531        /* XOR in (msn * remainder) [pre-rotated by 4 bits] */
 8532        z8[0] ^= ((word32)R[16 + (a & 0xf)]) << 16;
 8533
 8534        xi >>= 4;
 8535        /* XOR in next significant nibble (XORed with H) * remainder */
 8536        m8 = (word32*)m[xi];
 8537        a ^= (byte)(m8[3] >> 12) & 0xf;
 8538        a ^= (byte)((m8[3] << 4) & 0xf0);
 8539        z8[0] ^= ((word32)R[a >> 4]) << 16;
 8540
 8541        /* XOR in (next significant nibble * H) [pre-rotated by 4 bits] */
 8542        m8 = (word32*)m[16 + xi];
 8543        z8[0] ^= m8[0]; z8[1] ^= m8[1];
 8544        z8[2] ^= m8[2]; z8[3] ^= m8[3];
 8545    }
 8546
 8547    xi = x[0];
 8548
 8549    /* XOR in most significant nibble * H */
 8550    m8 = (word32*)m[xi & 0xf];
 8551    z8[0] ^= m8[0]; z8[1] ^= m8[1]; z8[2] ^= m8[2]; z8[3] ^= m8[3];
 8552
 8553    /* Cache top byte for remainder calculations - lost in rotate. */
 8554    a = (byte)(z8[3] & 0x0f);
 8555
 8556    z8[3] = (z8[2] << 28) | (z8[3] >> 4);
 8557    z8[2] = (z8[1] << 28) | (z8[2] >> 4);
 8558    z8[1] = (z8[0] << 28) | (z8[1] >> 4);
 8559    z8[0] >>= 4;
 8560
 8561    /* XOR in most significant nibble * remainder */
 8562    z8[0] ^= ((word32)R[a]) << 16;
 8563    /* XOR in next significant nibble * H */
 8564    m8 = (word32*)m[xi >> 4];
 8565    z8[0] ^= m8[0]; z8[1] ^= m8[1]; z8[2] ^= m8[2]; z8[3] ^= m8[3];
 8566
 8567    /* Write back result. */
 8568    x8[0] = z8[0]; x8[1] = z8[1]; x8[2] = z8[2]; x8[3] = z8[3];
 8569}
 8570#elif defined(WC_32BIT_CPU)
 8571static WC_INLINE void GMULT(byte *x, byte m[32][WC_AES_BLOCK_SIZE])
 8572{
 8573    int i;
 8574    word32 z8[4] = {0, 0, 0, 0};
 8575    byte a;
 8576    word32* x8 = (word32*)x;
 8577    word32* m8;
 8578    byte xi;
 8579    word32 n7, n6, n5, n4, n3, n2, n1, n0;
 8580
 8581    for (i = 15; i > 0; i--) {
 8582        xi = x[i];
 8583
 8584        /* XOR in (msn * H) */
 8585        m8 = (word32*)m[xi & 0xf];
 8586        z8[0] ^= m8[0]; z8[1] ^= m8[1]; z8[2] ^= m8[2]; z8[3] ^= m8[3];
 8587
 8588        /* Cache top byte for remainder calculations - lost in rotate. */
 8589        a = (byte)(z8[3] >> 24);
 8590
 8591        /* Rotate Z by 8-bits */
 8592        z8[3] = (z8[2] >> 24) | (z8[3] << 8);
 8593        z8[2] = (z8[1] >> 24) | (z8[2] << 8);
 8594        z8[1] = (z8[0] >> 24) | (z8[1] << 8);
 8595        z8[0] <<= 8;
 8596
 8597        /* XOR in (msn * remainder) [pre-rotated by 4 bits] */
 8598        z8[0] ^= (word32)R[16 + (a & 0xf)];
 8599
 8600        xi >>= 4;
 8601        /* XOR in next significant nibble (XORed with H) * remainder */
 8602        m8 = (word32*)m[xi];
 8603        a ^= (byte)(m8[3] >> 20);
 8604        z8[0] ^= (word32)R[a >> 4];
 8605
 8606        /* XOR in (next significant nibble * H) [pre-rotated by 4 bits] */
 8607        m8 = (word32*)m[16 + xi];
 8608        z8[0] ^= m8[0]; z8[1] ^= m8[1];
 8609        z8[2] ^= m8[2]; z8[3] ^= m8[3];
 8610    }
 8611
 8612    xi = x[0];
 8613
 8614    /* XOR in most significant nibble * H */
 8615    m8 = (word32*)m[xi & 0xf];
 8616    z8[0] ^= m8[0]; z8[1] ^= m8[1]; z8[2] ^= m8[2]; z8[3] ^= m8[3];
 8617
 8618    /* Cache top byte for remainder calculations - lost in rotate. */
 8619    a = (z8[3] >> 24) & 0xf;
 8620
 8621    /* Rotate z by 4-bits */
 8622    n7 = z8[3] & 0xf0f0f0f0ULL;
 8623    n6 = z8[3] & 0x0f0f0f0fULL;
 8624    n5 = z8[2] & 0xf0f0f0f0ULL;
 8625    n4 = z8[2] & 0x0f0f0f0fULL;
 8626    n3 = z8[1] & 0xf0f0f0f0ULL;
 8627    n2 = z8[1] & 0x0f0f0f0fULL;
 8628    n1 = z8[0] & 0xf0f0f0f0ULL;
 8629    n0 = z8[0] & 0x0f0f0f0fULL;
 8630    z8[3] = (n7 >> 4) | (n6 << 12) | (n4 >> 20);
 8631    z8[2] = (n5 >> 4) | (n4 << 12) | (n2 >> 20);
 8632    z8[1] = (n3 >> 4) | (n2 << 12) | (n0 >> 20);
 8633    z8[0] = (n1 >> 4) | (n0 << 12);
 8634
 8635    /* XOR in most significant nibble * remainder */
 8636    z8[0] ^= (word32)R[a];
 8637    /* XOR in next significant nibble * H */
 8638    m8 = (word32*)m[xi >> 4];
 8639    z8[0] ^= m8[0]; z8[1] ^= m8[1]; z8[2] ^= m8[2]; z8[3] ^= m8[3];
 8640
 8641    /* Write back result. */
 8642    x8[0] = z8[0]; x8[1] = z8[1]; x8[2] = z8[2]; x8[3] = z8[3];
 8643}
 8644#elif defined(WC_64BIT_CPU) && defined(BIG_ENDIAN_ORDER)
 8645static WC_INLINE void GMULT(byte *x, byte m[32][WC_AES_BLOCK_SIZE])
 8646{
 8647    int i;
 8648    word64 z8[2] = {0, 0};
 8649    byte a;
 8650    word64* x8 = (word64*)x;
 8651    word64* m8;
 8652    byte xi;
 8653
 8654    for (i = 15; i > 0; i--) {
 8655        xi = x[i];
 8656
 8657        /* XOR in (msn * H) */
 8658        m8 = (word64*)m[xi & 0xf];
 8659        z8[0] ^= m8[0];
 8660        z8[1] ^= m8[1];
 8661
 8662        /* Cache top byte for remainder calculations - lost in rotate. */
 8663        a = (byte)(z8[1] & 0xff);
 8664
 8665        /* Rotate Z by 8-bits */
 8666        z8[1] = (z8[0] << 56) | (z8[1] >> 8);
 8667        z8[0] >>= 8;
 8668
 8669        /* XOR in (next significant nibble * H) [pre-rotated by 4 bits] */
 8670        m8 = (word64*)m[16 + (xi >> 4)];
 8671        z8[0] ^= m8[0];
 8672        z8[1] ^= m8[1];
 8673
 8674        /* XOR in (msn * remainder) [pre-rotated by 4 bits] */
 8675        z8[0] ^= ((word64)R[16 + (a & 0xf)]) << 48;
 8676        /* XOR in next significant nibble (XORed with H) * remainder */
 8677        m8 = (word64*)m[xi >> 4];
 8678        a ^= (byte)(m8[1] >> 12) & 0xf;
 8679        a ^= (byte)((m8[1] << 4) & 0xf0);
 8680        z8[0] ^= ((word64)R[a >> 4]) << 48;
 8681    }
 8682
 8683    xi = x[0];
 8684
 8685    /* XOR in most significant nibble * H */
 8686    m8 = (word64*)m[xi & 0xf];
 8687    z8[0] ^= m8[0];
 8688    z8[1] ^= m8[1];
 8689
 8690    /* Cache top byte for remainder calculations - lost in rotate. */
 8691    a = (byte)(z8[1] & 0x0f);
 8692
 8693    /* Rotate z by 4-bits */
 8694    z8[1] = (z8[0] << 60) | (z8[1] >> 4);
 8695    z8[0] >>= 4;
 8696
 8697    /* XOR in next significant nibble * H */
 8698    m8 = (word64*)m[xi >> 4];
 8699    z8[0] ^= m8[0];
 8700    z8[1] ^= m8[1];
 8701    /* XOR in most significant nibble * remainder */
 8702    z8[0] ^= ((word64)R[a]) << 48;
 8703
 8704    /* Write back result. */
 8705    x8[0] = z8[0];
 8706    x8[1] = z8[1];
 8707}
 8708#else
 8709static WC_INLINE void GMULT(byte *x, byte m[32][WC_AES_BLOCK_SIZE])
 8710{
 8711    int i;
 8712    word64 z8[2] = {0, 0};
 8713    byte a;
 8714    word64* x8 = (word64*)x;
 8715    word64* m8;
 8716    word64 n0, n1, n2, n3;
 8717    byte xi;
 8718
 8719    for (i = 15; i > 0; i--) {
 8720        xi = x[i];
 8721
 8722        /* XOR in (msn * H) */
 8723        m8 = (word64*)m[xi & 0xf];
 8724        z8[0] ^= m8[0];
 8725        z8[1] ^= m8[1];
 8726
 8727        /* Cache top byte for remainder calculations - lost in rotate. */
 8728        a = (byte)(z8[1] >> 56);
 8729
 8730        /* Rotate Z by 8-bits */
 8731        z8[1] = (z8[0] >> 56) | (z8[1] << 8);
 8732        z8[0] <<= 8;
 8733
 8734        /* XOR in (next significant nibble * H) [pre-rotated by 4 bits] */
 8735        m8 = (word64*)m[16 + (xi >> 4)];
 8736        z8[0] ^= m8[0];
 8737        z8[1] ^= m8[1];
 8738
 8739        /* XOR in (msn * remainder) [pre-rotated by 4 bits] */
 8740        z8[0] ^= (word64)R[16 + (a & 0xf)];
 8741        /* XOR in next significant nibble (XORed with H) * remainder */
 8742        m8 = (word64*)m[xi >> 4];
 8743        a ^= (byte)(m8[1] >> 52);
 8744        z8[0] ^= (word64)R[a >> 4];
 8745    }
 8746
 8747    xi = x[0];
 8748
 8749    /* XOR in most significant nibble * H */
 8750    m8 = (word64*)m[xi & 0xf];
 8751    z8[0] ^= m8[0];
 8752    z8[1] ^= m8[1];
 8753
 8754    /* Cache top byte for remainder calculations - lost in rotate. */
 8755    a = (z8[1] >> 56) & 0xf;
 8756
 8757    /* Rotate z by 4-bits */
 8758    n3 = z8[1] & W64LIT(0xf0f0f0f0f0f0f0f0);
 8759    n2 = z8[1] & W64LIT(0x0f0f0f0f0f0f0f0f);
 8760    n1 = z8[0] & W64LIT(0xf0f0f0f0f0f0f0f0);
 8761    n0 = z8[0] & W64LIT(0x0f0f0f0f0f0f0f0f);
 8762    z8[1] = (n3 >> 4) | (n2 << 12) | (n0 >> 52);
 8763    z8[0] = (n1 >> 4) | (n0 << 12);
 8764
 8765    /* XOR in next significant nibble * H */
 8766    m8 = (word64*)m[xi >> 4];
 8767    z8[0] ^= m8[0];
 8768    z8[1] ^= m8[1];
 8769    /* XOR in most significant nibble * remainder */
 8770    z8[0] ^= (word64)R[a];
 8771
 8772    /* Write back result. */
 8773    x8[0] = z8[0];
 8774    x8[1] = z8[1];
 8775}
 8776#endif
 8777#endif
 8778
 8779void GHASH(Gcm* gcm, const byte* a, word32 aSz, const byte* c,
 8780    word32 cSz, byte* s, word32 sSz)
 8781{
 8782    byte x[WC_AES_BLOCK_SIZE];
 8783    byte scratch[WC_AES_BLOCK_SIZE];
 8784    word32 blocks, partial;
 8785
 8786    if (gcm == NULL) {
 8787        return;
 8788    }
 8789
 8790    XMEMSET(x, 0, WC_AES_BLOCK_SIZE);
 8791
 8792    /* Hash in A, the Additional Authentication Data */
 8793    if (aSz != 0 && a != NULL) {
 8794        blocks = aSz / WC_AES_BLOCK_SIZE;
 8795        partial = aSz % WC_AES_BLOCK_SIZE;
 8796    #ifdef GCM_GMULT_LEN
 8797        if (blocks > 0) {
 8798            GCM_GMULT_LEN(gcm, x, a, blocks * WC_AES_BLOCK_SIZE);
 8799            a += blocks * WC_AES_BLOCK_SIZE;
 8800        }
 8801        if (partial != 0) {
 8802            XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
 8803            XMEMCPY(scratch, a, partial);
 8804            GCM_GMULT_LEN(gcm, x, scratch, WC_AES_BLOCK_SIZE);
 8805        }
 8806    #else
 8807        while (blocks--) {
 8808            xorbuf(x, a, WC_AES_BLOCK_SIZE);
 8809            GMULT(x, gcm->M0);
 8810            a += WC_AES_BLOCK_SIZE;
 8811        }
 8812        if (partial != 0) {
 8813            XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
 8814            XMEMCPY(scratch, a, partial);
 8815            xorbuf(x, scratch, WC_AES_BLOCK_SIZE);
 8816            GMULT(x, gcm->M0);
 8817        }
 8818    #endif
 8819    }
 8820
 8821    /* Hash in C, the Ciphertext */
 8822    if (cSz != 0 && c != NULL) {
 8823        blocks = cSz / WC_AES_BLOCK_SIZE;
 8824        partial = cSz % WC_AES_BLOCK_SIZE;
 8825    #ifdef GCM_GMULT_LEN
 8826        if (blocks > 0) {
 8827            GCM_GMULT_LEN(gcm, x, c, blocks * WC_AES_BLOCK_SIZE);
 8828            c += blocks * WC_AES_BLOCK_SIZE;
 8829        }
 8830        if (partial != 0) {
 8831            XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
 8832            XMEMCPY(scratch, c, partial);
 8833            GCM_GMULT_LEN(gcm, x, scratch, WC_AES_BLOCK_SIZE);
 8834        }
 8835    #else
 8836        while (blocks--) {
 8837            xorbuf(x, c, WC_AES_BLOCK_SIZE);
 8838            GMULT(x, gcm->M0);
 8839            c += WC_AES_BLOCK_SIZE;
 8840        }
 8841        if (partial != 0) {
 8842            XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
 8843            XMEMCPY(scratch, c, partial);
 8844            xorbuf(x, scratch, WC_AES_BLOCK_SIZE);
 8845            GMULT(x, gcm->M0);
 8846        }
 8847    #endif
 8848    }
 8849
 8850    /* Hash in the lengths of A and C in bits */
 8851    FlattenSzInBits(&scratch[0], aSz);
 8852    FlattenSzInBits(&scratch[8], cSz);
 8853#ifdef GCM_GMULT_LEN
 8854    GCM_GMULT_LEN(gcm, x, scratch, WC_AES_BLOCK_SIZE);
 8855#else
 8856    xorbuf(x, scratch, WC_AES_BLOCK_SIZE);
 8857    GMULT(x, gcm->M0);
 8858#endif
 8859
 8860    /* Copy the result into s. */
 8861    XMEMCPY(s, x, sSz);
 8862}
 8863
 8864#ifdef WOLFSSL_AESGCM_STREAM
 8865/* No extra initialization for 4-bit table implementation.
 8866 *
 8867 * @param [in] aes  AES GCM object.
 8868 */
 8869#define GHASH_INIT_EXTRA(aes) WC_DO_NOTHING
 8870
 8871#ifdef GCM_GMULT_LEN
 8872/* GHASH one block of data.
 8873 *
 8874 * @param [in, out] aes    AES GCM object.
 8875 * @param [in]      block  Block of AAD or cipher text.
 8876 */
 8877#define GHASH_ONE_BLOCK_SW(aes, block)                                  \
 8878   GCM_GMULT_LEN(&(aes)->gcm, AES_TAG(aes), block, WC_AES_BLOCK_SIZE)
 8879#else
 8880/* GHASH one block of data.
 8881 *
 8882 * XOR block into tag and GMULT with H using pre-computed table.
 8883 *
 8884 * @param [in, out] aes    AES GCM object.
 8885 * @param [in]      block  Block of AAD or cipher text.
 8886 */
 8887#define GHASH_ONE_BLOCK_SW(aes, block)                  \
 8888    do {                                                \
 8889        xorbuf(AES_TAG(aes), block, WC_AES_BLOCK_SIZE); \
 8890        GMULT(AES_TAG(aes), (aes)->gcm.M0);             \
 8891    }                                                   \
 8892    while (0)
 8893#endif
 8894#endif /* WOLFSSL_AESGCM_STREAM */
 8895#elif defined(WORD64_AVAILABLE) && !defined(GCM_WORD32)
 8896
 8897#if !defined(FREESCALE_LTC_AES_GCM)
 8898static void GMULT(word64* X, word64* Y)
 8899{
 8900    word64 Z[2] = {0,0};
 8901    word64 V[2];
 8902    int i, j;
 8903    word64 v1;
 8904    V[0] = X[0];  V[1] = X[1];
 8905
 8906    for (i = 0; i < 2; i++)
 8907    {
 8908        word64 y = Y[i];
 8909        for (j = 0; j < 64; j++)
 8910        {
 8911#ifndef AES_GCM_GMULT_NCT
 8912            word64 mask = 0 - (y >> 63);
 8913            Z[0] ^= V[0] & mask;
 8914            Z[1] ^= V[1] & mask;
 8915#else
 8916            if (y & 0x8000000000000000ULL) {
 8917                Z[0] ^= V[0];
 8918                Z[1] ^= V[1];
 8919            }
 8920#endif
 8921
 8922            v1 = (0 - (V[1] & 1)) & 0xE100000000000000ULL;
 8923            V[1] >>= 1;
 8924            V[1] |= V[0] << 63;
 8925            V[0] >>= 1;
 8926            V[0] ^= v1;
 8927            y <<= 1;
 8928        }
 8929    }
 8930    X[0] = Z[0];
 8931    X[1] = Z[1];
 8932}
 8933
 8934
 8935void GHASH(Gcm* gcm, const byte* a, word32 aSz, const byte* c,
 8936    word32 cSz, byte* s, word32 sSz)
 8937{
 8938    word64 x[2] = {0,0};
 8939    word32 blocks, partial;
 8940    word64 bigH[2];
 8941
 8942    if (gcm == NULL) {
 8943        return;
 8944    }
 8945
 8946    XMEMCPY(bigH, gcm->H, WC_AES_BLOCK_SIZE);
 8947    #ifdef LITTLE_ENDIAN_ORDER
 8948        ByteReverseWords64(bigH, bigH, WC_AES_BLOCK_SIZE);
 8949    #endif
 8950
 8951    /* Hash in A, the Additional Authentication Data */
 8952    if (aSz != 0 && a != NULL) {
 8953        word64 bigA[2];
 8954        blocks = aSz / WC_AES_BLOCK_SIZE;
 8955        partial = aSz % WC_AES_BLOCK_SIZE;
 8956        while (blocks--) {
 8957            XMEMCPY(bigA, a, WC_AES_BLOCK_SIZE);
 8958            #ifdef LITTLE_ENDIAN_ORDER
 8959                ByteReverseWords64(bigA, bigA, WC_AES_BLOCK_SIZE);
 8960            #endif
 8961            x[0] ^= bigA[0];
 8962            x[1] ^= bigA[1];
 8963            GMULT(x, bigH);
 8964            a += WC_AES_BLOCK_SIZE;
 8965        }
 8966        if (partial != 0) {
 8967            XMEMSET(bigA, 0, WC_AES_BLOCK_SIZE);
 8968            XMEMCPY(bigA, a, partial);
 8969            #ifdef LITTLE_ENDIAN_ORDER
 8970                ByteReverseWords64(bigA, bigA, WC_AES_BLOCK_SIZE);
 8971            #endif
 8972            x[0] ^= bigA[0];
 8973            x[1] ^= bigA[1];
 8974            GMULT(x, bigH);
 8975        }
 8976#ifdef OPENSSL_EXTRA
 8977        /* store AAD partial tag for next call */
 8978        gcm->aadH[0] = (word32)((x[0] & 0xFFFFFFFF00000000ULL) >> 32);
 8979        gcm->aadH[1] = (word32)(x[0] & 0xFFFFFFFF);
 8980        gcm->aadH[2] = (word32)((x[1] & 0xFFFFFFFF00000000ULL) >> 32);
 8981        gcm->aadH[3] = (word32)(x[1] & 0xFFFFFFFF);
 8982#endif
 8983    }
 8984
 8985    /* Hash in C, the Ciphertext */
 8986    if (cSz != 0 && c != NULL) {
 8987        word64 bigC[2];
 8988        blocks = cSz / WC_AES_BLOCK_SIZE;
 8989        partial = cSz % WC_AES_BLOCK_SIZE;
 8990#ifdef OPENSSL_EXTRA
 8991        /* Start from last AAD partial tag */
 8992        if(gcm->aadLen) {
 8993            x[0] = ((word64)gcm->aadH[0]) << 32 | gcm->aadH[1];
 8994            x[1] = ((word64)gcm->aadH[2]) << 32 | gcm->aadH[3];
 8995         }
 8996#endif
 8997        while (blocks--) {
 8998            XMEMCPY(bigC, c, WC_AES_BLOCK_SIZE);
 8999            #ifdef LITTLE_ENDIAN_ORDER
 9000                ByteReverseWords64(bigC, bigC, WC_AES_BLOCK_SIZE);
 9001            #endif
 9002            x[0] ^= bigC[0];
 9003            x[1] ^= bigC[1];
 9004            GMULT(x, bigH);
 9005            c += WC_AES_BLOCK_SIZE;
 9006        }
 9007        if (partial != 0) {
 9008            XMEMSET(bigC, 0, WC_AES_BLOCK_SIZE);
 9009            XMEMCPY(bigC, c, partial);
 9010            #ifdef LITTLE_ENDIAN_ORDER
 9011                ByteReverseWords64(bigC, bigC, WC_AES_BLOCK_SIZE);
 9012            #endif
 9013            x[0] ^= bigC[0];
 9014            x[1] ^= bigC[1];
 9015            GMULT(x, bigH);
 9016        }
 9017    }
 9018
 9019    /* Hash in the lengths in bits of A and C */
 9020    {
 9021        word64 len[2];
 9022        len[0] = aSz; len[1] = cSz;
 9023#ifdef OPENSSL_EXTRA
 9024        if (gcm->aadLen)
 9025            len[0] = (word64)gcm->aadLen;
 9026#endif
 9027        /* Lengths are in bytes. Convert to bits. */
 9028        len[0] *= 8;
 9029        len[1] *= 8;
 9030
 9031        x[0] ^= len[0];
 9032        x[1] ^= len[1];
 9033        GMULT(x, bigH);
 9034    }
 9035    #ifdef LITTLE_ENDIAN_ORDER
 9036        ByteReverseWords64(x, x, WC_AES_BLOCK_SIZE);
 9037    #endif
 9038    XMEMCPY(s, x, sSz);
 9039}
 9040#endif /* !FREESCALE_LTC_AES_GCM */
 9041
 9042#ifdef WOLFSSL_AESGCM_STREAM
 9043
 9044#ifdef LITTLE_ENDIAN_ORDER
 9045
 9046/* No extra initialization for small implementation.
 9047 *
 9048 * @param [in] aes  AES GCM object.
 9049 */
 9050#define GHASH_INIT_EXTRA(aes)                                               \
 9051    ByteReverseWords64((word64*)aes->gcm.H, (word64*)aes->gcm.H, WC_AES_BLOCK_SIZE)
 9052
 9053/* GHASH one block of data..
 9054 *
 9055 * XOR block into tag and GMULT with H.
 9056 *
 9057 * @param [in, out] aes    AES GCM object.
 9058 * @param [in]      block  Block of AAD or cipher text.
 9059 */
 9060#define GHASH_ONE_BLOCK_SW(aes, block)                              \
 9061    do {                                                            \
 9062        word64* x = (word64*)AES_TAG(aes);                          \
 9063        word64* h = (word64*)aes->gcm.H;                            \
 9064        word64 block64[2];                                          \
 9065        XMEMCPY(block64, block, WC_AES_BLOCK_SIZE);                 \
 9066        ByteReverseWords64(block64, block64, WC_AES_BLOCK_SIZE);    \
 9067        x[0] ^= block64[0];                                         \
 9068        x[1] ^= block64[1];                                         \
 9069        GMULT(x, h);                                                \
 9070    }                                                               \
 9071    while (0)
 9072
 9073#ifdef OPENSSL_EXTRA
 9074/* GHASH in AAD and cipher text lengths in bits.
 9075 *
 9076 * Convert tag back to little-endian.
 9077 *
 9078 * @param [in, out] aes  AES GCM object.
 9079 */
 9080#define GHASH_LEN_BLOCK(aes)                            \
 9081    do {                                                \
 9082        word64* x = (word64*)AES_TAG(aes);              \
 9083        word64* h = (word64*)aes->gcm.H;                \
 9084        word64 len[2];                                  \
 9085        len[0] = aes->aSz; len[1] = aes->cSz;           \
 9086        if (aes->gcm.aadLen)                            \
 9087            len[0] = (word64)aes->gcm.aadLen;           \
 9088        /* Lengths are in bytes. Convert to bits. */    \
 9089        len[0] *= 8;                                    \
 9090        len[1] *= 8;                                    \
 9091                                                        \
 9092        x[0] ^= len[0];                                 \
 9093        x[1] ^= len[1];                                 \
 9094        GMULT(x, h);                                    \
 9095        ByteReverseWords64(x, x, WC_AES_BLOCK_SIZE);    \
 9096    }                                                   \
 9097    while (0)
 9098#else
 9099/* GHASH in AAD and cipher text lengths in bits.
 9100 *
 9101 * Convert tag back to little-endian.
 9102 *
 9103 * @param [in, out] aes  AES GCM object.
 9104 */
 9105#define GHASH_LEN_BLOCK(aes)                            \
 9106    do {                                                \
 9107        word64* x = (word64*)AES_TAG(aes);              \
 9108        word64* h = (word64*)aes->gcm.H;                \
 9109        word64 len[2];                                  \
 9110        len[0] = aes->aSz; len[1] = aes->cSz;           \
 9111        /* Lengths are in bytes. Convert to bits. */    \
 9112        len[0] *= 8;                                    \
 9113        len[1] *= 8;                                    \
 9114                                                        \
 9115        x[0] ^= len[0];                                 \
 9116        x[1] ^= len[1];                                 \
 9117        GMULT(x, h);                                    \
 9118        ByteReverseWords64(x, x, WC_AES_BLOCK_SIZE);    \
 9119    }                                                   \
 9120    while (0)
 9121#endif
 9122
 9123#else
 9124
 9125/* No extra initialization for small implementation.
 9126 *
 9127 * @param [in] aes  AES GCM object.
 9128 */
 9129#define GHASH_INIT_EXTRA(aes) WC_DO_NOTHING
 9130
 9131/* GHASH one block of data..
 9132 *
 9133 * XOR block into tag and GMULT with H.
 9134 *
 9135 * @param [in, out] aes    AES GCM object.
 9136 * @param [in]      block  Block of AAD or cipher text.
 9137 */
 9138#define GHASH_ONE_BLOCK_SW(aes, block)                  \
 9139    do {                                                \
 9140        word64* x = (word64*)AES_TAG(aes);              \
 9141        word64* h = (word64*)aes->gcm.H;                \
 9142        word64 block64[2];                              \
 9143        XMEMCPY(block64, block, WC_AES_BLOCK_SIZE);        \
 9144        x[0] ^= block64[0];                             \
 9145        x[1] ^= block64[1];                             \
 9146        GMULT(x, h);                                    \
 9147    }                                                   \
 9148    while (0)
 9149
 9150#ifdef OPENSSL_EXTRA
 9151/* GHASH in AAD and cipher text lengths in bits.
 9152 *
 9153 * Convert tag back to little-endian.
 9154 *
 9155 * @param [in, out] aes  AES GCM object.
 9156 */
 9157#define GHASH_LEN_BLOCK(aes)                            \
 9158    do {                                                \
 9159        word64* x = (word64*)AES_TAG(aes);              \
 9160        word64* h = (word64*)aes->gcm.H;                \
 9161        word64 len[2];                                  \
 9162        len[0] = aes->aSz; len[1] = aes->cSz;           \
 9163        if (aes->gcm.aadLen)                            \
 9164            len[0] = (word64)aes->gcm.aadLen;           \
 9165        /* Lengths are in bytes. Convert to bits. */    \
 9166        len[0] *= 8;                                    \
 9167        len[1] *= 8;                                    \
 9168                                                        \
 9169        x[0] ^= len[0];                                 \
 9170        x[1] ^= len[1];                                 \
 9171        GMULT(x, h);                                    \
 9172    }                                                   \
 9173    while (0)
 9174#else
 9175/* GHASH in AAD and cipher text lengths in bits.
 9176 *
 9177 * Convert tag back to little-endian.
 9178 *
 9179 * @param [in, out] aes  AES GCM object.
 9180 */
 9181#define GHASH_LEN_BLOCK(aes)                            \
 9182    do {                                                \
 9183        word64* x = (word64*)AES_TAG(aes);              \
 9184        word64* h = (word64*)aes->gcm.H;                \
 9185        word64 len[2];                                  \
 9186        len[0] = aes->aSz; len[1] = aes->cSz;           \
 9187        /* Lengths are in bytes. Convert to bits. */    \
 9188        len[0] *= 8;                                    \
 9189        len[1] *= 8;                                    \
 9190                                                        \
 9191        x[0] ^= len[0];                                 \
 9192        x[1] ^= len[1];                                 \
 9193        GMULT(x, h);                                    \
 9194    }                                                   \
 9195    while (0)
 9196#endif
 9197
 9198#endif /* !LITTLE_ENDIAN_ORDER */
 9199
 9200#endif /* WOLFSSL_AESGCM_STREAM */
 9201/* end defined(WORD64_AVAILABLE) && !defined(GCM_WORD32) */
 9202#else /* GCM_WORD32 */
 9203
 9204static void GMULT(word32* X, word32* Y)
 9205{
 9206    word32 Z[4] = {0,0,0,0};
 9207    word32 V[4];
 9208    int i, j;
 9209
 9210    V[0] = X[0];  V[1] = X[1]; V[2] =  X[2]; V[3] =  X[3];
 9211
 9212    for (i = 0; i < 4; i++)
 9213    {
 9214        word32 y = Y[i];
 9215        for (j = 0; j < 32; j++)
 9216        {
 9217            if (y & 0x80000000) {
 9218                Z[0] ^= V[0];
 9219                Z[1] ^= V[1];
 9220                Z[2] ^= V[2];
 9221                Z[3] ^= V[3];
 9222            }
 9223
 9224            if (V[3] & 0x00000001) {
 9225                V[3] >>= 1;
 9226                V[3] |= ((V[2] & 0x00000001) ? 0x80000000 : 0);
 9227                V[2] >>= 1;
 9228                V[2] |= ((V[1] & 0x00000001) ? 0x80000000 : 0);
 9229                V[1] >>= 1;
 9230                V[1] |= ((V[0] & 0x00000001) ? 0x80000000 : 0);
 9231                V[0] >>= 1;
 9232                V[0] ^= 0xE1000000;
 9233            } else {
 9234                V[3] >>= 1;
 9235                V[3] |= ((V[2] & 0x00000001) ? 0x80000000 : 0);
 9236                V[2] >>= 1;
 9237                V[2] |= ((V[1] & 0x00000001) ? 0x80000000 : 0);
 9238                V[1] >>= 1;
 9239                V[1] |= ((V[0] & 0x00000001) ? 0x80000000 : 0);
 9240                V[0] >>= 1;
 9241            }
 9242            y <<= 1;
 9243        }
 9244    }
 9245    X[0] = Z[0];
 9246    X[1] = Z[1];
 9247    X[2] = Z[2];
 9248    X[3] = Z[3];
 9249}
 9250
 9251
 9252void GHASH(Gcm* gcm, const byte* a, word32 aSz, const byte* c,
 9253    word32 cSz, byte* s, word32 sSz)
 9254{
 9255    word32 x[4] = {0,0,0,0};
 9256    word32 blocks, partial;
 9257    word32 bigH[4];
 9258
 9259    if (gcm == NULL) {
 9260        return;
 9261    }
 9262
 9263    XMEMCPY(bigH, gcm->H, WC_AES_BLOCK_SIZE);
 9264    #ifdef LITTLE_ENDIAN_ORDER
 9265        ByteReverseWords(bigH, bigH, WC_AES_BLOCK_SIZE);
 9266    #endif
 9267
 9268    /* Hash in A, the Additional Authentication Data */
 9269    if (aSz != 0 && a != NULL) {
 9270        word32 bigA[4];
 9271        blocks = aSz / WC_AES_BLOCK_SIZE;
 9272        partial = aSz % WC_AES_BLOCK_SIZE;
 9273        while (blocks--) {
 9274            XMEMCPY(bigA, a, WC_AES_BLOCK_SIZE);
 9275            #ifdef LITTLE_ENDIAN_ORDER
 9276                ByteReverseWords(bigA, bigA, WC_AES_BLOCK_SIZE);
 9277            #endif
 9278            x[0] ^= bigA[0];
 9279            x[1] ^= bigA[1];
 9280            x[2] ^= bigA[2];
 9281            x[3] ^= bigA[3];
 9282            GMULT(x, bigH);
 9283            a += WC_AES_BLOCK_SIZE;
 9284        }
 9285        if (partial != 0) {
 9286            XMEMSET(bigA, 0, WC_AES_BLOCK_SIZE);
 9287            XMEMCPY(bigA, a, partial);
 9288            #ifdef LITTLE_ENDIAN_ORDER
 9289                ByteReverseWords(bigA, bigA, WC_AES_BLOCK_SIZE);
 9290            #endif
 9291            x[0] ^= bigA[0];
 9292            x[1] ^= bigA[1];
 9293            x[2] ^= bigA[2];
 9294            x[3] ^= bigA[3];
 9295            GMULT(x, bigH);
 9296        }
 9297    }
 9298
 9299    /* Hash in C, the Ciphertext */
 9300    if (cSz != 0 && c != NULL) {
 9301        word32 bigC[4];
 9302        blocks = cSz / WC_AES_BLOCK_SIZE;
 9303        partial = cSz % WC_AES_BLOCK_SIZE;
 9304        while (blocks--) {
 9305            XMEMCPY(bigC, c, WC_AES_BLOCK_SIZE);
 9306            #ifdef LITTLE_ENDIAN_ORDER
 9307                ByteReverseWords(bigC, bigC, WC_AES_BLOCK_SIZE);
 9308            #endif
 9309            x[0] ^= bigC[0];
 9310            x[1] ^= bigC[1];
 9311            x[2] ^= bigC[2];
 9312            x[3] ^= bigC[3];
 9313            GMULT(x, bigH);
 9314            c += WC_AES_BLOCK_SIZE;
 9315        }
 9316        if (partial != 0) {
 9317            XMEMSET(bigC, 0, WC_AES_BLOCK_SIZE);
 9318            XMEMCPY(bigC, c, partial);
 9319            #ifdef LITTLE_ENDIAN_ORDER
 9320                ByteReverseWords(bigC, bigC, WC_AES_BLOCK_SIZE);
 9321            #endif
 9322            x[0] ^= bigC[0];
 9323            x[1] ^= bigC[1];
 9324            x[2] ^= bigC[2];
 9325            x[3] ^= bigC[3];
 9326            GMULT(x, bigH);
 9327        }
 9328    }
 9329
 9330    /* Hash in the lengths in bits of A and C */
 9331    {
 9332        word32 len[4];
 9333
 9334        /* Lengths are in bytes. Convert to bits. */
 9335        len[0] = (aSz >> (8*sizeof(aSz) - 3));
 9336        len[1] = aSz << 3;
 9337        len[2] = (cSz >> (8*sizeof(cSz) - 3));
 9338        len[3] = cSz << 3;
 9339
 9340        x[0] ^= len[0];
 9341        x[1] ^= len[1];
 9342        x[2] ^= len[2];
 9343        x[3] ^= len[3];
 9344        GMULT(x, bigH);
 9345    }
 9346    #ifdef LITTLE_ENDIAN_ORDER
 9347        ByteReverseWords(x, x, WC_AES_BLOCK_SIZE);
 9348    #endif
 9349    XMEMCPY(s, x, sSz);
 9350}
 9351
 9352#ifdef WOLFSSL_AESGCM_STREAM
 9353#ifdef LITTLE_ENDIAN_ORDER
 9354/* Little-endian 32-bit word implementation requires byte reversal of H.
 9355 *
 9356 * H is all-zeros block encrypted with key.
 9357 *
 9358 * @param [in, out] aes  AES GCM object.
 9359 */
 9360#define GHASH_INIT_EXTRA(aes) \
 9361    ByteReverseWords((word32*)aes->gcm.H, (word32*)aes->gcm.H, WC_AES_BLOCK_SIZE)
 9362
 9363/* GHASH one block of data..
 9364 *
 9365 * XOR block, in big-endian form, into tag and GMULT with H.
 9366 *
 9367 * @param [in, out] aes    AES GCM object.
 9368 * @param [in]      block  Block of AAD or cipher text.
 9369 */
 9370#define GHASH_ONE_BLOCK_SW(aes, block)                          \
 9371    do {                                                        \
 9372        word32* x = (word32*)AES_TAG(aes);                      \
 9373        word32* h = (word32*)aes->gcm.H;                        \
 9374        word32 bigEnd[4];                                       \
 9375        XMEMCPY(bigEnd, block, WC_AES_BLOCK_SIZE);              \
 9376        ByteReverseWords(bigEnd, bigEnd, WC_AES_BLOCK_SIZE);    \
 9377        x[0] ^= bigEnd[0];                                      \
 9378        x[1] ^= bigEnd[1];                                      \
 9379        x[2] ^= bigEnd[2];                                      \
 9380        x[3] ^= bigEnd[3];                                      \
 9381        GMULT(x, h);                                            \
 9382    }                                                           \
 9383    while (0)
 9384
 9385/* GHASH in AAD and cipher text lengths in bits.
 9386 *
 9387 * Convert tag back to little-endian.
 9388 *
 9389 * @param [in, out] aes  AES GCM object.
 9390 */
 9391#define GHASH_LEN_BLOCK(aes)                                \
 9392    do {                                                    \
 9393        word32 len[4];                                      \
 9394        word32* x = (word32*)AES_TAG(aes);                  \
 9395        word32* h = (word32*)aes->gcm.H;                    \
 9396        len[0] = (aes->aSz >> (8*sizeof(aes->aSz) - 3));    \
 9397        len[1] = aes->aSz << 3;                             \
 9398        len[2] = (aes->cSz >> (8*sizeof(aes->cSz) - 3));    \
 9399        len[3] = aes->cSz << 3;                             \
 9400        x[0] ^= len[0];                                     \
 9401        x[1] ^= len[1];                                     \
 9402        x[2] ^= len[2];                                     \
 9403        x[3] ^= len[3];                                     \
 9404        GMULT(x, h);                                        \
 9405        ByteReverseWords(x, x, WC_AES_BLOCK_SIZE);          \
 9406    }                                                       \
 9407    while (0)
 9408#else
 9409/* No extra initialization for 32-bit word implementation.
 9410 *
 9411 * @param [in] aes  AES GCM object.
 9412 */
 9413#define GHASH_INIT_EXTRA(aes) WC_DO_NOTHING
 9414
 9415/* GHASH one block of data..
 9416 *
 9417 * XOR block into tag and GMULT with H.
 9418 *
 9419 * @param [in, out] aes    AES GCM object.
 9420 * @param [in]      block  Block of AAD or cipher text.
 9421 */
 9422#define GHASH_ONE_BLOCK_SW(aes, block)                      \
 9423    do {                                                    \
 9424        word32* x = (word32*)AES_TAG(aes);                  \
 9425        word32* h = (word32*)aes->gcm.H;                    \
 9426        word32 block32[4];                                  \
 9427        XMEMCPY(block32, block, WC_AES_BLOCK_SIZE);         \
 9428        x[0] ^= block32[0];                                 \
 9429        x[1] ^= block32[1];                                 \
 9430        x[2] ^= block32[2];                                 \
 9431        x[3] ^= block32[3];                                 \
 9432        GMULT(x, h);                                        \
 9433    }                                                       \
 9434    while (0)
 9435
 9436/* GHASH in AAD and cipher text lengths in bits.
 9437 *
 9438 * @param [in, out] aes  AES GCM object.
 9439 */
 9440#define GHASH_LEN_BLOCK(aes)                                \
 9441    do {                                                    \
 9442        word32 len[4];                                      \
 9443        word32* x = (word32*)AES_TAG(aes);                  \
 9444        word32* h = (word32*)aes->gcm.H;                    \
 9445        len[0] = (aes->aSz >> (8*sizeof(aes->aSz) - 3));    \
 9446        len[1] = aes->aSz << 3;                             \
 9447        len[2] = (aes->cSz >> (8*sizeof(aes->cSz) - 3));    \
 9448        len[3] = aes->cSz << 3;                             \
 9449        x[0] ^= len[0];                                     \
 9450        x[1] ^= len[1];                                     \
 9451        x[2] ^= len[2];                                     \
 9452        x[3] ^= len[3];                                     \
 9453        GMULT(x, h);                                        \
 9454    }                                                       \
 9455    while (0)
 9456#endif /* LITTLE_ENDIAN_ORDER */
 9457#endif /* WOLFSSL_AESGCM_STREAM */
 9458#endif /* end GCM_WORD32 */
 9459#endif
 9460
 9461#if !defined(WOLFSSL_XILINX_CRYPT) && !defined(WOLFSSL_AFALG_XILINX_AES)
 9462#ifdef WOLFSSL_AESGCM_STREAM
 9463#ifndef GHASH_LEN_BLOCK
 9464/* Hash in the lengths of the AAD and cipher text in bits.
 9465 *
 9466 * Default implementation.
 9467 *
 9468 * @param [in, out] aes  AES GCM object.
 9469 */
 9470#define GHASH_LEN_BLOCK(aes)                      \
 9471    do {                                          \
 9472        byte scratch[WC_AES_BLOCK_SIZE];          \
 9473        FlattenSzInBits(&scratch[0], (aes)->aSz); \
 9474        FlattenSzInBits(&scratch[8], (aes)->cSz); \
 9475        GHASH_ONE_BLOCK(aes, scratch);            \
 9476    }                                             \
 9477    while (0)
 9478#endif
 9479
 9480/* Initialize a GHASH for streaming operations.
 9481 *
 9482 * @param [in, out] aes  AES GCM object.
 9483 */
 9484static void GHASH_INIT(Aes* aes) {
 9485    /* Set tag to all zeros as initial value. */
 9486    XMEMSET(AES_TAG(aes), 0, WC_AES_BLOCK_SIZE);
 9487    /* Reset counts of AAD and cipher text. */
 9488    aes->aOver = 0;
 9489    aes->cOver = 0;
 9490#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
 9491    !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
 9492    if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) {
 9493        ; /* Don't do extra initialization. */
 9494    }
 9495    else
 9496#endif
 9497    {
 9498        /* Extra initialization based on implementation. */
 9499        GHASH_INIT_EXTRA(aes);
 9500    }
 9501}
 9502
 9503/* Update the GHASH with AAD and/or cipher text.
 9504 *
 9505 * @param [in,out] aes   AES GCM object.
 9506 * @param [in]     a     Additional authentication data buffer.
 9507 * @param [in]     aSz   Size of data in AAD buffer.
 9508 * @param [in]     c     Cipher text buffer.
 9509 * @param [in]     cSz   Size of data in cipher text buffer.
 9510 */
 9511static void GHASH_UPDATE(Aes* aes, const byte* a, word32 aSz, const byte* c,
 9512    word32 cSz)
 9513{
 9514    word32 blocks;
 9515    word32 partial;
 9516
 9517    /* Hash in A, the Additional Authentication Data */
 9518    if (aSz != 0 && a != NULL) {
 9519        /* Update count of AAD we have hashed. */
 9520        aes->aSz += aSz;
 9521        /* Check if we have unprocessed data. */
 9522        if (aes->aOver > 0) {
 9523            /* Calculate amount we can use - fill up the block. */
 9524            byte sz = (byte)(WC_AES_BLOCK_SIZE - aes->aOver);
 9525            if (sz > aSz) {
 9526                sz = (byte)aSz;
 9527            }
 9528            /* Copy extra into last GHASH block array and update count. */
 9529            XMEMCPY(AES_LASTGBLOCK(aes) + aes->aOver, a, sz);
 9530            aes->aOver = (byte)(aes->aOver + sz);
 9531            if (aes->aOver == WC_AES_BLOCK_SIZE) {
 9532                /* We have filled up the block and can process. */
 9533                GHASH_ONE_BLOCK(aes, AES_LASTGBLOCK(aes));
 9534                /* Reset count. */
 9535                aes->aOver = 0;
 9536            }
 9537            /* Used up some data. */
 9538            aSz -= sz;
 9539            a += sz;
 9540        }
 9541
 9542        /* Calculate number of blocks of AAD and the leftover. */
 9543        blocks = aSz / WC_AES_BLOCK_SIZE;
 9544        partial = aSz % WC_AES_BLOCK_SIZE;
 9545        /* GHASH full blocks now. */
 9546        while (blocks--) {
 9547            GHASH_ONE_BLOCK(aes, a);
 9548            a += WC_AES_BLOCK_SIZE;
 9549        }
 9550        if (partial != 0) {
 9551            /* Cache the partial block. */
 9552            XMEMCPY(AES_LASTGBLOCK(aes), a, partial);
 9553            aes->aOver = (byte)partial;
 9554        }
 9555    }
 9556    if (aes->aOver > 0 && cSz > 0 && c != NULL) {
 9557        /* No more AAD coming and we have a partial block. */
 9558        /* Fill the rest of the block with zeros. */
 9559        byte sz = (byte)(WC_AES_BLOCK_SIZE - aes->aOver);
 9560        XMEMSET(AES_LASTGBLOCK(aes) + aes->aOver, 0, sz);
 9561        /* GHASH last AAD block. */
 9562        GHASH_ONE_BLOCK(aes, AES_LASTGBLOCK(aes));
 9563        /* Clear partial count for next time through. */
 9564        aes->aOver = 0;
 9565    }
 9566
 9567    /* Hash in C, the Ciphertext */
 9568    if (cSz != 0 && c != NULL) {
 9569        /* Update count of cipher text we have hashed. */
 9570        aes->cSz += cSz;
 9571        if (aes->cOver > 0) {
 9572            /* Calculate amount we can use - fill up the block. */
 9573            byte sz = (byte)(WC_AES_BLOCK_SIZE - aes->cOver);
 9574            if (sz > cSz) {
 9575                sz = (byte)cSz;
 9576            }
 9577            XMEMCPY(AES_LASTGBLOCK(aes) + aes->cOver, c, sz);
 9578            /* Update count of unused encrypted counter. */
 9579            aes->cOver = (byte)(aes->cOver + sz);
 9580            if (aes->cOver == WC_AES_BLOCK_SIZE) {
 9581                /* We have filled up the block and can process. */
 9582                GHASH_ONE_BLOCK(aes, AES_LASTGBLOCK(aes));
 9583                /* Reset count. */
 9584                aes->cOver = 0;
 9585            }
 9586            /* Used up some data. */
 9587            cSz -= sz;
 9588            c += sz;
 9589        }
 9590
 9591        /* Calculate number of blocks of cipher text and the leftover. */
 9592        blocks = cSz / WC_AES_BLOCK_SIZE;
 9593        partial = cSz % WC_AES_BLOCK_SIZE;
 9594        /* GHASH full blocks now. */
 9595        while (blocks--) {
 9596            GHASH_ONE_BLOCK(aes, c);
 9597            c += WC_AES_BLOCK_SIZE;
 9598        }
 9599        if (partial != 0) {
 9600            /* Cache the partial block. */
 9601            XMEMCPY(AES_LASTGBLOCK(aes), c, partial);
 9602            aes->cOver = (byte)partial;
 9603        }
 9604    }
 9605}
 9606
 9607/* Finalize the GHASH calculation.
 9608 *
 9609 * Complete hashing cipher text and hash the AAD and cipher text lengths.
 9610 *
 9611 * @param [in, out] aes  AES GCM object.
 9612 * @param [out]     s    Authentication tag.
 9613 * @param [in]      sSz  Size of authentication tag required.
 9614 */
 9615static void GHASH_FINAL(Aes* aes, byte* s, word32 sSz)
 9616{
 9617    /* AAD block incomplete when > 0 */
 9618    byte over = aes->aOver;
 9619
 9620    if (aes->cOver > 0) {
 9621        /* Cipher text block incomplete. */
 9622        over = aes->cOver;
 9623    }
 9624    if (over > 0) {
 9625        /* Zeroize the unused part of the block. */
 9626        XMEMSET(AES_LASTGBLOCK(aes) + over, 0,
 9627            (size_t)WC_AES_BLOCK_SIZE - over);
 9628        /* Hash the last block of cipher text. */
 9629        GHASH_ONE_BLOCK(aes, AES_LASTGBLOCK(aes));
 9630    }
 9631    /* Hash in the lengths of AAD and cipher text in bits */
 9632    GHASH_LEN_BLOCK(aes);
 9633    /* Copy the result into s. */
 9634    XMEMCPY(s, AES_TAG(aes), sSz);
 9635    /* reset aes->gcm.H in case of reuse */
 9636    GHASH_INIT_EXTRA(aes);
 9637}
 9638#endif /* WOLFSSL_AESGCM_STREAM */
 9639
 9640
 9641#ifdef FREESCALE_LTC_AES_GCM
 9642int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz,
 9643                   const byte* iv, word32 ivSz,
 9644                   byte* authTag, word32 authTagSz,
 9645                   const byte* authIn, word32 authInSz)
 9646{
 9647    status_t status;
 9648    word32 keySize;
 9649
 9650    /* argument checks */
 9651    if (aes == NULL || authTagSz > WC_AES_BLOCK_SIZE || ivSz == 0) {
 9652        return BAD_FUNC_ARG;
 9653    }
 9654
 9655    if (authTagSz < WOLFSSL_MIN_AUTH_TAG_SZ) {
 9656        WOLFSSL_MSG("GcmEncrypt authTagSz too small error");
 9657        return BAD_FUNC_ARG;
 9658    }
 9659
 9660    status = wc_AesGetKeySize(aes, &keySize);
 9661    if (status)
 9662        return status;
 9663
 9664    status = wolfSSL_CryptHwMutexLock();
 9665    if (status != 0)
 9666        return status;
 9667
 9668    status = LTC_AES_EncryptTagGcm(LTC_BASE, in, out, sz, iv, ivSz,
 9669        authIn, authInSz, (byte*)aes->key, keySize, authTag, authTagSz);
 9670    wolfSSL_CryptHwMutexUnLock();
 9671
 9672    return (status == kStatus_Success) ? 0 : AES_GCM_AUTH_E;
 9673}
 9674
 9675#else
 9676
 9677#ifdef STM32_CRYPTO_AES_GCM
 9678
 9679/* this function supports inline encrypt */
 9680static WARN_UNUSED_RESULT int wc_AesGcmEncrypt_STM32(
 9681                                  Aes* aes, byte* out, const byte* in, word32 sz,
 9682                                  const byte* iv, word32 ivSz,
 9683                                  byte* authTag, word32 authTagSz,
 9684                                  const byte* authIn, word32 authInSz)
 9685{
 9686    int ret;
 9687#ifdef WOLFSSL_STM32_CUBEMX
 9688    CRYP_HandleTypeDef hcryp;
 9689#else
 9690    word32 keyCopy[AES_256_KEY_SIZE/sizeof(word32)];
 9691#endif
 9692    word32 keySize;
 9693#ifdef WOLFSSL_STM32_CUBEMX
 9694    int status = HAL_OK;
 9695    word32 blocks = sz / WC_AES_BLOCK_SIZE;
 9696    word32 partialBlock[WC_AES_BLOCK_SIZE/sizeof(word32)];
 9697#else
 9698    int status = SUCCESS;
 9699#endif
 9700    word32 partial = sz % WC_AES_BLOCK_SIZE;
 9701    word32 tag[WC_AES_BLOCK_SIZE/sizeof(word32)];
 9702    word32 ctrInit[WC_AES_BLOCK_SIZE/sizeof(word32)];
 9703    word32 ctr[WC_AES_BLOCK_SIZE/sizeof(word32)];
 9704    word32 authhdr[WC_AES_BLOCK_SIZE/sizeof(word32)];
 9705    byte* authInPadded = NULL;
 9706    int authPadSz, wasAlloc = 0, useSwGhash = 0;
 9707
 9708    ret = wc_AesGetKeySize(aes, &keySize);
 9709    if (ret != 0)
 9710        return ret;
 9711
 9712#ifdef WOLFSSL_STM32_CUBEMX
 9713    ret = wc_Stm32_Aes_Init(aes, &hcryp, 0);
 9714    if (ret != 0)
 9715        return ret;
 9716#endif
 9717
 9718    XMEMSET(ctr, 0, WC_AES_BLOCK_SIZE);
 9719    if (ivSz == GCM_NONCE_MID_SZ) {
 9720        byte* pCtr = (byte*)ctr;
 9721        XMEMCPY(ctr, iv, ivSz);
 9722        pCtr[WC_AES_BLOCK_SIZE - 1] = 1;
 9723    }
 9724    else {
 9725        GHASH(&aes->gcm, NULL, 0, iv, ivSz, (byte*)ctr, WC_AES_BLOCK_SIZE);
 9726    }
 9727    XMEMCPY(ctrInit, ctr, sizeof(ctr)); /* save off initial counter for GMAC */
 9728
 9729    /* Authentication buffer */
 9730#if STM_CRYPT_HEADER_WIDTH == 1
 9731    authPadSz = 0; /* CubeHAL supports byte mode */
 9732#else
 9733    authPadSz = authInSz % STM_CRYPT_HEADER_WIDTH;
 9734#endif
 9735#ifdef WOLFSSL_STM32MP13
 9736    /* STM32MP13 HAL at least v1.2 and lower has a bug with which it needs a
 9737     * minimum of 16 bytes for the auth */
 9738    if ((authInSz > 0) && (authInSz < 16)) {
 9739        authPadSz = 16 - authInSz;
 9740    }
 9741#endif
 9742    if (authPadSz != 0) {
 9743        if (authPadSz < authInSz + STM_CRYPT_HEADER_WIDTH) {
 9744            authPadSz = authInSz + STM_CRYPT_HEADER_WIDTH - authPadSz;
 9745        }
 9746        if (authPadSz <= sizeof(authhdr)) {
 9747            authInPadded = (byte*)authhdr;
 9748        }
 9749        else {
 9750            authInPadded = (byte*)XMALLOC(authPadSz, aes->heap,
 9751                DYNAMIC_TYPE_TMP_BUFFER);
 9752            if (authInPadded == NULL) {
 9753                wolfSSL_CryptHwMutexUnLock();
 9754                return MEMORY_E;
 9755            }
 9756            wasAlloc = 1;
 9757        }
 9758        XMEMSET(authInPadded, 0, authPadSz);
 9759        XMEMCPY(authInPadded, authIn, authInSz);
 9760    } else {
 9761        authPadSz = authInSz;
 9762        authInPadded = (byte*)authIn;
 9763    }
 9764
 9765    /* for cases where hardware cannot be used for authTag calculate it */
 9766    /* if IV is not 12 calculate GHASH using software */
 9767    if (ivSz != GCM_NONCE_MID_SZ
 9768    #if !defined(CRYP_HEADERWIDTHUNIT_BYTE)
 9769        /* or hardware that does not support partial block */
 9770        || sz == 0 || partial != 0
 9771    #endif
 9772    #if STM_CRYPT_HEADER_WIDTH == 4
 9773        /* or authIn is not a multiple of 4  */
 9774        || authPadSz != authInSz
 9775    #endif
 9776    ) {
 9777        useSwGhash = 1;
 9778    }
 9779
 9780    /* Hardware requires counter + 1 */
 9781    IncrementGcmCounter((byte*)ctr);
 9782
 9783    ret = wolfSSL_CryptHwMutexLock();
 9784    if (ret != 0) {
 9785        return ret;
 9786    }
 9787
 9788#ifdef WOLFSSL_STM32_CUBEMX
 9789    hcryp.Init.pInitVect = (STM_CRYPT_TYPE*)ctr;
 9790    hcryp.Init.Header = (STM_CRYPT_TYPE*)authInPadded;
 9791
 9792#if defined(STM32_HAL_V2)
 9793    hcryp.Init.Algorithm = CRYP_AES_GCM;
 9794    hcryp.Init.HeaderSize = authPadSz / STM_CRYPT_HEADER_WIDTH;
 9795    #ifdef CRYP_KEYIVCONFIG_ONCE
 9796    /* allows repeated calls to HAL_CRYP_Encrypt */
 9797    hcryp.Init.KeyIVConfigSkip = CRYP_KEYIVCONFIG_ONCE;
 9798    #endif
 9799    ByteReverseWords(ctr, ctr, WC_AES_BLOCK_SIZE);
 9800    hcryp.Init.pInitVect = (STM_CRYPT_TYPE*)ctr;
 9801    HAL_CRYP_Init(&hcryp);
 9802
 9803    #ifndef CRYP_KEYIVCONFIG_ONCE
 9804    /* GCM payload phase - can handle partial blocks */
 9805    status = HAL_CRYP_Encrypt(&hcryp, (uint32_t*)in,
 9806        (blocks * WC_AES_BLOCK_SIZE) + partial, (uint32_t*)out, STM32_HAL_TIMEOUT);
 9807    #else
 9808    /* GCM payload phase - blocks */
 9809    if (blocks) {
 9810        status = HAL_CRYP_Encrypt(&hcryp, (uint32_t*)in,
 9811            (blocks * WC_AES_BLOCK_SIZE), (uint32_t*)out, STM32_HAL_TIMEOUT);
 9812    }
 9813    /* GCM payload phase - partial remainder */
 9814    if (status == HAL_OK && (partial != 0 || blocks == 0)) {
 9815        XMEMSET(partialBlock, 0, sizeof(partialBlock));
 9816        XMEMCPY(partialBlock, in + (blocks * WC_AES_BLOCK_SIZE), partial);
 9817        status = HAL_CRYP_Encrypt(&hcryp, (uint32_t*)partialBlock, partial,
 9818            (uint32_t*)partialBlock, STM32_HAL_TIMEOUT);
 9819        XMEMCPY(out + (blocks * WC_AES_BLOCK_SIZE), partialBlock, partial);
 9820    }
 9821    #endif
 9822    if (status == HAL_OK && !useSwGhash) {
 9823        /* Compute the authTag */
 9824        status = HAL_CRYPEx_AESGCM_GenerateAuthTAG(&hcryp, (uint32_t*)tag,
 9825            STM32_HAL_TIMEOUT);
 9826    }
 9827#elif defined(STM32_CRYPTO_AES_ONLY)
 9828    /* Set the CRYP parameters */
 9829    hcryp.Init.HeaderSize = authPadSz;
 9830    if (authPadSz == 0)
 9831        hcryp.Init.Header = NULL; /* cannot pass pointer when authIn == 0 */
 9832    hcryp.Init.ChainingMode  = CRYP_CHAINMODE_AES_GCM_GMAC;
 9833    hcryp.Init.OperatingMode = CRYP_ALGOMODE_ENCRYPT;
 9834    hcryp.Init.GCMCMACPhase  = CRYP_INIT_PHASE;
 9835    HAL_CRYP_Init(&hcryp);
 9836
 9837    /* GCM init phase */
 9838    status = HAL_CRYPEx_AES_Auth(&hcryp, NULL, 0, NULL, STM32_HAL_TIMEOUT);
 9839    if (status == HAL_OK) {
 9840        /* GCM header phase */
 9841        hcryp.Init.GCMCMACPhase = CRYP_HEADER_PHASE;
 9842        status = HAL_CRYPEx_AES_Auth(&hcryp, NULL, 0, NULL, STM32_HAL_TIMEOUT);
 9843    }
 9844    if (status == HAL_OK) {
 9845        /* GCM payload phase - blocks */
 9846        hcryp.Init.GCMCMACPhase = CRYP_PAYLOAD_PHASE;
 9847        if (blocks) {
 9848            status = HAL_CRYPEx_AES_Auth(&hcryp, (byte*)in,
 9849                (blocks * WC_AES_BLOCK_SIZE), out, STM32_HAL_TIMEOUT);
 9850        }
 9851    }
 9852    if (status == HAL_OK && (partial != 0 || (sz > 0 && blocks == 0))) {
 9853        /* GCM payload phase - partial remainder */
 9854        XMEMSET(partialBlock, 0, sizeof(partialBlock));
 9855        XMEMCPY(partialBlock, in + (blocks * WC_AES_BLOCK_SIZE), partial);
 9856        status = HAL_CRYPEx_AES_Auth(&hcryp, (uint8_t*)partialBlock, partial,
 9857                (uint8_t*)partialBlock, STM32_HAL_TIMEOUT);
 9858        XMEMCPY(out + (blocks * WC_AES_BLOCK_SIZE), partialBlock, partial);
 9859    }
 9860    if (status == HAL_OK && !useSwGhash) {
 9861        /* GCM final phase */
 9862        hcryp.Init.GCMCMACPhase  = CRYP_FINAL_PHASE;
 9863        status = HAL_CRYPEx_AES_Auth(&hcryp, NULL, sz, (uint8_t*)tag, STM32_HAL_TIMEOUT);
 9864    }
 9865#else
 9866    hcryp.Init.HeaderSize = authPadSz;
 9867    HAL_CRYP_Init(&hcryp);
 9868    if (blocks) {
 9869        /* GCM payload phase - blocks */
 9870        status = HAL_CRYPEx_AESGCM_Encrypt(&hcryp, (byte*)in,
 9871            (blocks * WC_AES_BLOCK_SIZE), out, STM32_HAL_TIMEOUT);
 9872    }
 9873    if (status == HAL_OK && (partial != 0 || blocks == 0)) {
 9874        /* GCM payload phase - partial remainder */
 9875        XMEMSET(partialBlock, 0, sizeof(partialBlock));
 9876        XMEMCPY(partialBlock, in + (blocks * WC_AES_BLOCK_SIZE), partial);
 9877        status = HAL_CRYPEx_AESGCM_Encrypt(&hcryp, (uint8_t*)partialBlock, partial,
 9878            (uint8_t*)partialBlock, STM32_HAL_TIMEOUT);
 9879        XMEMCPY(out + (blocks * WC_AES_BLOCK_SIZE), partialBlock, partial);
 9880    }
 9881    if (status == HAL_OK && !useSwGhash) {
 9882        /* Compute the authTag */
 9883        status = HAL_CRYPEx_AESGCM_Finish(&hcryp, sz, (uint8_t*)tag, STM32_HAL_TIMEOUT);
 9884    }
 9885#endif
 9886
 9887    if (status != HAL_OK)
 9888        ret = AES_GCM_AUTH_E;
 9889    HAL_CRYP_DeInit(&hcryp);
 9890
 9891#else /* Standard Peripheral Library */
 9892    ByteReverseWords(keyCopy, (word32*)aes->key, keySize);
 9893    status = CRYP_AES_GCM(MODE_ENCRYPT, (uint8_t*)ctr,
 9894                         (uint8_t*)keyCopy,      keySize * 8,
 9895                         (uint8_t*)in,           sz,
 9896                         (uint8_t*)authInPadded, authInSz,
 9897                         (uint8_t*)out,          (uint8_t*)tag);
 9898    if (status != SUCCESS)
 9899        ret = AES_GCM_AUTH_E;
 9900#endif /* WOLFSSL_STM32_CUBEMX */
 9901    wolfSSL_CryptHwMutexUnLock();
 9902    wc_Stm32_Aes_Cleanup();
 9903
 9904    if (ret == 0) {
 9905        /* return authTag */
 9906        if (authTag) {
 9907            if (useSwGhash) {
 9908                GHASH(&aes->gcm, authIn, authInSz, out, sz, authTag, authTagSz);
 9909                ret = wc_AesEncrypt(aes, (byte*)ctrInit, (byte*)tag);
 9910                if (ret == 0) {
 9911                    xorbuf(authTag, tag, authTagSz);
 9912                }
 9913            }
 9914            else {
 9915                /* use hardware calculated tag */
 9916                XMEMCPY(authTag, tag, authTagSz);
 9917            }
 9918        }
 9919    }
 9920
 9921    /* Free memory */
 9922    if (wasAlloc) {
 9923        XFREE(authInPadded, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
 9924    }
 9925
 9926    return ret;
 9927}
 9928
 9929#endif /* STM32_CRYPTO_AES_GCM */
 9930
 9931#if !defined(WOLFSSL_ARMASM)
 9932#ifdef WOLFSSL_AESNI
 9933/* For performance reasons, this code needs to be not inlined. */
 9934WARN_UNUSED_RESULT int AES_GCM_encrypt_C(
 9935                      Aes* aes, byte* out, const byte* in, word32 sz,
 9936                      const byte* iv, word32 ivSz,
 9937                      byte* authTag, word32 authTagSz,
 9938                      const byte* authIn, word32 authInSz);
 9939#else
 9940static
 9941#endif
 9942WARN_UNUSED_RESULT int AES_GCM_encrypt_C(
 9943                      Aes* aes, byte* out, const byte* in, word32 sz,
 9944                      const byte* iv, word32 ivSz,
 9945                      byte* authTag, word32 authTagSz,
 9946                      const byte* authIn, word32 authInSz)
 9947{
 9948    int ret = 0;
 9949    word32 blocks = sz / WC_AES_BLOCK_SIZE;
 9950    word32 partial = sz % WC_AES_BLOCK_SIZE;
 9951    const byte* p = in;
 9952    byte* c = out;
 9953    ALIGN16 byte counter[WC_AES_BLOCK_SIZE];
 9954    ALIGN16 byte initialCounter[WC_AES_BLOCK_SIZE];
 9955    ALIGN16 byte scratch[WC_AES_BLOCK_SIZE];
 9956#ifdef WC_AES_HAVE_PREFETCH_ARG
 9957    int did_prefetches = 0;
 9958#endif
 9959
 9960    if (ivSz == GCM_NONCE_MID_SZ) {
 9961        /* Counter is IV with bottom 4 bytes set to: 0x00,0x00,0x00,0x01. */
 9962        XMEMCPY(counter, iv, ivSz);
 9963        XMEMSET(counter + GCM_NONCE_MID_SZ, 0,
 9964                                         WC_AES_BLOCK_SIZE - GCM_NONCE_MID_SZ - 1);
 9965        counter[WC_AES_BLOCK_SIZE - 1] = 1;
 9966    }
 9967    else {
 9968        /* Counter is GHASH of IV. */
 9969#ifdef OPENSSL_EXTRA
 9970        word32 aadTemp = aes->gcm.aadLen;
 9971        aes->gcm.aadLen = 0;
 9972#endif
 9973        GHASH(&aes->gcm, NULL, 0, iv, ivSz, counter, WC_AES_BLOCK_SIZE);
 9974#ifdef OPENSSL_EXTRA
 9975        aes->gcm.aadLen = aadTemp;
 9976#endif
 9977    }
 9978    XMEMCPY(initialCounter, counter, WC_AES_BLOCK_SIZE);
 9979
 9980#ifdef WOLFSSL_PIC32MZ_CRYPT
 9981    if (blocks) {
 9982        /* use initial IV for HW, but don't use it below */
 9983        XMEMCPY(aes->reg, counter, WC_AES_BLOCK_SIZE);
 9984
 9985        ret = wc_Pic32AesCrypt(
 9986            aes->key, aes->keylen, aes->reg, WC_AES_BLOCK_SIZE,
 9987            out, in, (blocks * WC_AES_BLOCK_SIZE),
 9988            PIC32_ENCRYPTION, PIC32_ALGO_AES, PIC32_CRYPTOALGO_AES_GCM);
 9989        if (ret != 0)
 9990            return ret;
 9991    }
 9992    /* process remainder using partial handling */
 9993#endif
 9994
 9995#if defined(HAVE_AES_ECB) && !defined(WOLFSSL_PIC32MZ_CRYPT)
 9996    /* some hardware acceleration can gain performance from doing AES encryption
 9997     * of the whole buffer at once */
 9998    if (c != p && blocks > 0) { /* can not handle inline encryption */
 9999        while (blocks--) {
10000            IncrementGcmCounter(counter);
10001            XMEMCPY(c, counter, WC_AES_BLOCK_SIZE);
10002            c += WC_AES_BLOCK_SIZE;
10003        }
10004
10005        /* reset number of blocks and then do encryption */
10006        blocks = sz / WC_AES_BLOCK_SIZE;
10007        wc_AesEcbEncrypt(aes, out, out, WC_AES_BLOCK_SIZE * blocks);
10008        xorbuf(out, p, WC_AES_BLOCK_SIZE * blocks);
10009        p += WC_AES_BLOCK_SIZE * blocks;
10010    }
10011    else
10012#endif /* HAVE_AES_ECB && !WOLFSSL_PIC32MZ_CRYPT */
10013    {
10014        while (blocks--) {
10015            IncrementGcmCounter(counter);
10016        #if !defined(WOLFSSL_PIC32MZ_CRYPT)
10017            ret = AesEncrypt_preFetchOpt(aes, counter, scratch,
10018                                            &did_prefetches);
10019            if (ret != 0)
10020                return ret;
10021            xorbufout(c, scratch, p, WC_AES_BLOCK_SIZE);
10022        #endif
10023            p += WC_AES_BLOCK_SIZE;
10024            c += WC_AES_BLOCK_SIZE;
10025        }
10026    }
10027
10028    if (partial != 0) {
10029        IncrementGcmCounter(counter);
10030        ret = AesEncrypt_preFetchOpt(aes, counter, scratch, &did_prefetches);
10031        if (ret != 0)
10032            return ret;
10033        xorbufout(c, scratch, p, partial);
10034    }
10035    if (authTag) {
10036        GHASH(&aes->gcm, authIn, authInSz, out, sz, authTag, authTagSz);
10037        ret = AesEncrypt_preFetchOpt(aes, initialCounter, scratch,
10038                                        &did_prefetches);
10039        if (ret != 0)
10040            return ret;
10041        xorbuf(authTag, scratch, authTagSz);
10042#ifdef OPENSSL_EXTRA
10043        if (!in && !sz)
10044            /* store AAD size for next call */
10045            aes->gcm.aadLen = authInSz;
10046#endif
10047    }
10048
10049    return ret;
10050}
10051#elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
10052static int AES_GCM_encrypt_ARM(Aes* aes, byte* out, const byte* in,
10053    word32 sz, const byte* iv, word32 ivSz, byte* authTag, word32 authTagSz,
10054    const byte* authIn, word32 authInSz)
10055{
10056    word32 blocks;
10057    word32 partial;
10058    byte counter[WC_AES_BLOCK_SIZE];
10059    byte initialCounter[WC_AES_BLOCK_SIZE];
10060    byte x[WC_AES_BLOCK_SIZE];
10061    byte scratch[WC_AES_BLOCK_SIZE];
10062
10063    XMEMSET(initialCounter, 0, WC_AES_BLOCK_SIZE);
10064    if (ivSz == GCM_NONCE_MID_SZ) {
10065        XMEMCPY(initialCounter, iv, ivSz);
10066        initialCounter[WC_AES_BLOCK_SIZE - 1] = 1;
10067    }
10068    else {
10069        GHASH(&aes->gcm, NULL, 0, iv, ivSz, initialCounter, WC_AES_BLOCK_SIZE);
10070    }
10071    XMEMCPY(counter, initialCounter, WC_AES_BLOCK_SIZE);
10072
10073    /* Hash in the Additional Authentication Data */
10074    XMEMSET(x, 0, WC_AES_BLOCK_SIZE);
10075    if (authInSz != 0 && authIn != NULL) {
10076        blocks = authInSz / WC_AES_BLOCK_SIZE;
10077        partial = authInSz % WC_AES_BLOCK_SIZE;
10078        if (blocks > 0) {
10079            GCM_GMULT_LEN(&aes->gcm, x, authIn, blocks * WC_AES_BLOCK_SIZE);
10080            authIn += blocks * WC_AES_BLOCK_SIZE;
10081        }
10082        if (partial != 0) {
10083            XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
10084            XMEMCPY(scratch, authIn, partial);
10085            GCM_GMULT_LEN(&aes->gcm, x, scratch, WC_AES_BLOCK_SIZE);
10086        }
10087    }
10088
10089    /* do as many blocks as possible */
10090    blocks = sz / WC_AES_BLOCK_SIZE;
10091    partial = sz % WC_AES_BLOCK_SIZE;
10092    if (blocks > 0) {
10093    #if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON)
10094    #ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
10095        if (sz >= 32)
10096    #endif
10097        {
10098            AES_GCM_encrypt_NEON(in, out, blocks * WC_AES_BLOCK_SIZE,
10099                (const unsigned char*)aes->key, aes->rounds, counter);
10100        }
10101    #ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
10102        else
10103    #endif
10104    #endif
10105    #ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
10106        {
10107            AES_GCM_encrypt(in, out, blocks * WC_AES_BLOCK_SIZE,
10108                (const unsigned char*)aes->key, aes->rounds, counter);
10109        }
10110    #endif
10111        GCM_GMULT_LEN(&aes->gcm, x, out, blocks * WC_AES_BLOCK_SIZE);
10112        in += blocks * WC_AES_BLOCK_SIZE;
10113        out += blocks * WC_AES_BLOCK_SIZE;
10114    }
10115    /* take care of partial block sizes leftover */
10116    if (partial != 0) {
10117    #if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON) && \
10118        defined(WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP)
10119        {
10120            AES_GCM_encrypt_NEON(in, scratch, WC_AES_BLOCK_SIZE,
10121                (const unsigned char*)aes->key, aes->rounds, counter);
10122        }
10123    #else
10124        {
10125            AES_GCM_encrypt(in, scratch, WC_AES_BLOCK_SIZE,
10126                (const unsigned char*)aes->key, aes->rounds, counter);
10127        }
10128    #endif
10129        XMEMCPY(out, scratch, partial);
10130
10131        XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
10132        XMEMCPY(scratch, out, partial);
10133        GCM_GMULT_LEN(&aes->gcm, x, scratch, WC_AES_BLOCK_SIZE);
10134    }
10135
10136    /* Hash in the lengths of A and C in bits */
10137    XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
10138    FlattenSzInBits(&scratch[0], authInSz);
10139    FlattenSzInBits(&scratch[8], sz);
10140    GCM_GMULT_LEN(&aes->gcm, x, scratch, WC_AES_BLOCK_SIZE);
10141    if (authTagSz > WC_AES_BLOCK_SIZE) {
10142        XMEMCPY(authTag, x, WC_AES_BLOCK_SIZE);
10143    }
10144    else {
10145        /* authTagSz can be smaller than WC_AES_BLOCK_SIZE */
10146        XMEMCPY(authTag, x, authTagSz);
10147    }
10148
10149    /* Auth tag calculation. */
10150#if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON) && \
10151    defined(WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP)
10152    {
10153        AES_ECB_encrypt_NEON(initialCounter, scratch, WC_AES_BLOCK_SIZE,
10154            (const unsigned char*)aes->key, aes->rounds);
10155    }
10156#else
10157    {
10158        AES_ECB_encrypt(initialCounter, scratch, WC_AES_BLOCK_SIZE,
10159            (const unsigned char*)aes->key, aes->rounds);
10160    }
10161#endif
10162    xorbuf(authTag, scratch, authTagSz);
10163
10164    return 0;
10165}
10166#endif
10167
10168/* Software AES - GCM Encrypt */
10169int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz,
10170                   const byte* iv, word32 ivSz,
10171                   byte* authTag, word32 authTagSz,
10172                   const byte* authIn, word32 authInSz)
10173{
10174    int ret;
10175
10176    /* argument checks */
10177    /* If sz is non-zero, both in and out must be set; if sz is 0, in and
10178     * out are don't cares (GMAC case), matching wc_AesGcmDecrypt. */
10179    if (aes == NULL || iv == NULL || ivSz == 0 ||
10180        (sz != 0 && (in == NULL || out == NULL)) ||
10181        authTag == NULL || authTagSz > WC_AES_BLOCK_SIZE ||
10182        ((authInSz > 0) && (authIn == NULL)))
10183    {
10184        return BAD_FUNC_ARG;
10185    }
10186
10187    if (authTagSz < WOLFSSL_MIN_AUTH_TAG_SZ) {
10188        WOLFSSL_MSG("GcmEncrypt authTagSz too small error");
10189        return BAD_FUNC_ARG;
10190    }
10191
10192#ifdef WOLF_CRYPTO_CB
10193    #ifndef WOLF_CRYPTO_CB_FIND
10194    if (aes->devId != INVALID_DEVID)
10195    #endif
10196    {
10197        int crypto_cb_ret =
10198            wc_CryptoCb_AesGcmEncrypt(aes, out, in, sz, iv, ivSz, authTag,
10199                                      authTagSz, authIn, authInSz);
10200        if (crypto_cb_ret != WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE))
10201            return crypto_cb_ret;
10202        /* fall-through when unavailable */
10203    }
10204#endif
10205
10206#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)
10207    /* if async and byte count above threshold */
10208    /* only 12-byte IV is supported in HW */
10209    if (aes->asyncDev.marker == WOLFSSL_ASYNC_MARKER_AES &&
10210                    sz >= WC_ASYNC_THRESH_AES_GCM && ivSz == GCM_NONCE_MID_SZ) {
10211    #if defined(HAVE_CAVIUM)
10212        #ifdef HAVE_CAVIUM_V
10213        if (authInSz == 20) { /* Nitrox V GCM is only working with 20 byte AAD */
10214            return NitroxAesGcmEncrypt(aes, out, in, sz,
10215                (const byte*)aes->devKey, aes->keylen, iv, ivSz,
10216                authTag, authTagSz, authIn, authInSz);
10217        }
10218        #endif
10219    #elif defined(HAVE_INTEL_QA)
10220        return IntelQaSymAesGcmEncrypt(&aes->asyncDev, out, in, sz,
10221            (const byte*)aes->devKey, aes->keylen, iv, ivSz,
10222            authTag, authTagSz, authIn, authInSz);
10223    #elif defined(WOLFSSL_ASYNC_CRYPT_SW)
10224        if (wc_AsyncSwInit(&aes->asyncDev, ASYNC_SW_AES_GCM_ENCRYPT)) {
10225            WC_ASYNC_SW* sw = &aes->asyncDev.sw;
10226            sw->aes.aes = aes;
10227            sw->aes.out = out;
10228            sw->aes.in = in;
10229            sw->aes.sz = sz;
10230            sw->aes.iv = iv;
10231            sw->aes.ivSz = ivSz;
10232            sw->aes.authTag = authTag;
10233            sw->aes.authTagSz = authTagSz;
10234            sw->aes.authIn = authIn;
10235            sw->aes.authInSz = authInSz;
10236            return WC_PENDING_E;
10237        }
10238    #endif
10239    }
10240#endif /* WOLFSSL_ASYNC_CRYPT */
10241
10242#ifdef WOLFSSL_SILABS_SE_ACCEL
10243    return wc_AesGcmEncrypt_silabs(
10244        aes, out, in, sz,
10245        iv, ivSz,
10246        authTag, authTagSz,
10247        authIn, authInSz);
10248#endif
10249#if defined(WOLFSSL_MICROCHIP_TA100) && defined(WOLFSSL_MICROCHIP_AESGCM)
10250#ifndef TA_AES_GCM_MAX_DATA_SIZE
10251    #define TA_AES_GCM_MAX_DATA_SIZE 996u
10252#endif
10253    if (aes != NULL &&
10254        aes->keylen == TA_KEY_TYPE_AES128_SIZE &&
10255        ivSz == TA_AES_GCM_IV_LENGTH &&
10256        authTagSz == TA_AES_GCM_TAG_LENGTH &&
10257        sz <= TA_AES_GCM_MAX_DATA_SIZE &&
10258        authInSz <= (word32)(TA_AES_GCM_MAX_DATA_SIZE - sz)) {
10259        return wc_Microchip_AesGcmEncrypt(
10260            aes, out, in, sz,
10261            iv, ivSz,
10262            authTag, authTagSz,
10263            authIn, authInSz);
10264    }
10265#endif
10266#ifdef STM32_CRYPTO_AES_GCM
10267    return wc_AesGcmEncrypt_STM32(
10268        aes, out, in, sz, iv, ivSz,
10269        authTag, authTagSz, authIn, authInSz);
10270#endif /* STM32_CRYPTO_AES_GCM */
10271
10272#if defined(WOLFSSL_PSOC6_CRYPTO)
10273    return wc_Psoc6_Aes_GcmEncrypt(aes, out, in, sz, iv, ivSz, authTag,
10274                                   authTagSz, authIn, authInSz);
10275#endif /* WOLFSSL_PSOC6_CRYPTO */
10276
10277    VECTOR_REGISTERS_PUSH;
10278
10279#if defined(WOLFSSL_ARMASM)
10280#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
10281#if !defined(__aarch64__)
10282    AES_GCM_encrypt_AARCH32(in, out, sz, iv, ivSz, authTag, authTagSz, authIn,
10283        authInSz, (byte*)aes->key, aes->gcm.H, (byte*)aes->tmp, (byte*)aes->reg,
10284        aes->rounds);
10285    ret = 0;
10286#else
10287    if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) {
10288    #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
10289        if (aes->use_sha3_hw_crypto) {
10290            AES_GCM_encrypt_AARCH64_EOR3(in, out, sz, iv, ivSz, authTag,
10291                authTagSz, authIn, authInSz, (byte*)aes->key, aes->gcm.H,
10292                (byte*)aes->tmp, (byte*)aes->reg, aes->rounds);
10293        }
10294        else
10295    #endif
10296        {
10297            AES_GCM_encrypt_AARCH64(in, out, sz, iv, ivSz, authTag, authTagSz,
10298                authIn, authInSz, (byte*)aes->key, aes->gcm.H, (byte*)aes->tmp,
10299                (byte*)aes->reg, aes->rounds);
10300        }
10301        ret = 0;
10302    }
10303    else
10304#endif /* !__aarch64__ */
10305#endif /* !WOLFSSL_ARMASM_NO_HW_CRYPTO */
10306#if defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
10307    {
10308        ret = AES_GCM_encrypt_ARM(aes, out, in, sz, iv, ivSz, authTag,
10309            authTagSz, authIn, authInSz);
10310    }
10311#endif /* __aarch64__ || WOLFSSL_ARMASM_NO_HW_CRYPTO */
10312#else
10313#ifdef WOLFSSL_AESNI
10314    if (aes->use_aesni) {
10315#ifdef HAVE_INTEL_AVX2
10316        if (IS_INTEL_AVX2(intel_flags)) {
10317            AES_GCM_encrypt_avx2(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
10318                                 authTagSz, (const byte*)aes->key, (int)aes->rounds);
10319            ret = 0;
10320        }
10321        else
10322#endif
10323#if defined(HAVE_INTEL_AVX1)
10324        if (IS_INTEL_AVX1(intel_flags)) {
10325            AES_GCM_encrypt_avx1(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
10326                                 authTagSz, (const byte*)aes->key, (int)aes->rounds);
10327            ret = 0;
10328        } else
10329#endif
10330        {
10331            AES_GCM_encrypt_aesni(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
10332                            authTagSz, (const byte*)aes->key, (int)aes->rounds);
10333            ret = 0;
10334        }
10335    }
10336    else
10337#endif /* WOLFSSL_AESNI */
10338    {
10339        ret = AES_GCM_encrypt_C(aes, out, in, sz, iv, ivSz, authTag, authTagSz,
10340                                authIn, authInSz);
10341    }
10342#endif
10343
10344    VECTOR_REGISTERS_POP;
10345
10346    return ret;
10347}
10348#endif
10349
10350
10351/* AES GCM Decrypt */
10352#if defined(HAVE_AES_DECRYPT) || defined(HAVE_AESGCM_DECRYPT)
10353#ifdef FREESCALE_LTC_AES_GCM
10354int  wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
10355                   const byte* iv, word32 ivSz,
10356                   const byte* authTag, word32 authTagSz,
10357                   const byte* authIn, word32 authInSz)
10358{
10359    int ret;
10360    word32 keySize;
10361    status_t status;
10362
10363    /* argument checks */
10364    /* If the sz is non-zero, both in and out must be set. If sz is 0,
10365     * in and out are don't cares, as this is is the GMAC case. */
10366    if (aes == NULL || iv == NULL || (sz != 0 && (in == NULL || out == NULL)) ||
10367        authTag == NULL || authTagSz > WC_AES_BLOCK_SIZE ||
10368        authTagSz < WOLFSSL_MIN_AUTH_TAG_SZ || ivSz == 0 ||
10369        ((authInSz > 0) && (authIn == NULL)))
10370    {
10371        return BAD_FUNC_ARG;
10372    }
10373
10374    ret = wc_AesGetKeySize(aes, &keySize);
10375    if (ret != 0) {
10376        return ret;
10377    }
10378
10379    status = wolfSSL_CryptHwMutexLock();
10380    if (status != 0)
10381        return status;
10382
10383    status = LTC_AES_DecryptTagGcm(LTC_BASE, in, out, sz, iv, ivSz,
10384        authIn, authInSz, (byte*)aes->key, keySize, authTag, authTagSz);
10385    wolfSSL_CryptHwMutexUnLock();
10386
10387    return (status == kStatus_Success) ? 0 : AES_GCM_AUTH_E;
10388}
10389
10390#else
10391
10392#ifdef STM32_CRYPTO_AES_GCM
10393/* this function supports inline decrypt */
10394static WARN_UNUSED_RESULT int wc_AesGcmDecrypt_STM32(
10395                                  Aes* aes, byte* out,
10396                                  const byte* in, word32 sz,
10397                                  const byte* iv, word32 ivSz,
10398                                  const byte* authTag, word32 authTagSz,
10399                                  const byte* authIn, word32 authInSz)
10400{
10401    int ret;
10402#ifdef WOLFSSL_STM32_CUBEMX
10403    int status = HAL_OK;
10404    CRYP_HandleTypeDef hcryp;
10405    word32 blocks = sz / WC_AES_BLOCK_SIZE;
10406#else
10407    int status = SUCCESS;
10408    word32 keyCopy[AES_256_KEY_SIZE/sizeof(word32)];
10409#endif
10410    word32 keySize;
10411    word32 partial = sz % WC_AES_BLOCK_SIZE;
10412    word32 tag[WC_AES_BLOCK_SIZE/sizeof(word32)];
10413    word32 tagExpected[WC_AES_BLOCK_SIZE/sizeof(word32)];
10414    word32 partialBlock[WC_AES_BLOCK_SIZE/sizeof(word32)];
10415    word32 ctr[WC_AES_BLOCK_SIZE/sizeof(word32)];
10416    word32 authhdr[WC_AES_BLOCK_SIZE/sizeof(word32)];
10417    byte* authInPadded = NULL;
10418    int authPadSz, wasAlloc = 0, tagComputed = 0;
10419
10420    ret = wc_AesGetKeySize(aes, &keySize);
10421    if (ret != 0)
10422        return ret;
10423
10424#ifdef WOLFSSL_STM32_CUBEMX
10425    ret = wc_Stm32_Aes_Init(aes, &hcryp, 0);
10426    if (ret != 0)
10427        return ret;
10428#endif
10429
10430    XMEMSET(ctr, 0, WC_AES_BLOCK_SIZE);
10431    if (ivSz == GCM_NONCE_MID_SZ) {
10432        byte* pCtr = (byte*)ctr;
10433        XMEMCPY(ctr, iv, ivSz);
10434        pCtr[WC_AES_BLOCK_SIZE - 1] = 1;
10435    }
10436    else {
10437        GHASH(&aes->gcm, NULL, 0, iv, ivSz, (byte*)ctr, WC_AES_BLOCK_SIZE);
10438    }
10439
10440    /* Make copy of expected authTag, which could get corrupted in some
10441     * Cube HAL versions without proper partial block support.
10442     * For TLS blocks the authTag is after the output buffer, so save it */
10443    XMEMCPY(tagExpected, authTag, authTagSz);
10444
10445    /* Authentication buffer */
10446#if STM_CRYPT_HEADER_WIDTH == 1
10447    authPadSz = 0; /* CubeHAL supports byte mode */
10448#else
10449    authPadSz = authInSz % STM_CRYPT_HEADER_WIDTH;
10450#endif
10451#ifdef WOLFSSL_STM32MP13
10452    /* STM32MP13 HAL at least v1.2 and lower has a bug with which it needs a
10453     * minimum of 16 bytes for the auth */
10454    if ((authInSz > 0) && (authInSz < 16)) {
10455        authPadSz = 16 - authInSz;
10456    }
10457#else
10458    if (authPadSz != 0) {
10459        authPadSz = authInSz + STM_CRYPT_HEADER_WIDTH - authPadSz;
10460    }
10461    else {
10462        authPadSz = authInSz;
10463    }
10464#endif
10465
10466    /* for cases where hardware cannot be used for authTag calculate it */
10467    /* if IV is not 12 calculate GHASH using software */
10468    if (ivSz != GCM_NONCE_MID_SZ
10469    #if !defined(CRYP_HEADERWIDTHUNIT_BYTE)
10470        /* or hardware that does not support partial block */
10471        || sz == 0 || partial != 0
10472    #endif
10473    #if STM_CRYPT_HEADER_WIDTH == 4
10474        /* or authIn is not a multiple of 4  */
10475        || authPadSz != authInSz
10476    #endif
10477    ) {
10478        GHASH(&aes->gcm, authIn, authInSz, in, sz, (byte*)tag, sizeof(tag));
10479        ret = wc_AesEncrypt(aes, (byte*)ctr, (byte*)partialBlock);
10480        if (ret != 0)
10481            return ret;
10482        xorbuf(tag, partialBlock, sizeof(tag));
10483        tagComputed = 1;
10484    }
10485
10486    /* if using hardware for authentication tag make sure its aligned and zero padded */
10487    if (authPadSz != authInSz && !tagComputed) {
10488        if (authPadSz <= sizeof(authhdr)) {
10489            authInPadded = (byte*)authhdr;
10490        }
10491        else {
10492            authInPadded = (byte*)XMALLOC(authPadSz, aes->heap,
10493                DYNAMIC_TYPE_TMP_BUFFER);
10494            if (authInPadded == NULL) {
10495                wolfSSL_CryptHwMutexUnLock();
10496                return MEMORY_E;
10497            }
10498            wasAlloc = 1;
10499        }
10500        XMEMSET(authInPadded, 0, authPadSz);
10501        XMEMCPY(authInPadded, authIn, authInSz);
10502    } else {
10503        authInPadded = (byte*)authIn;
10504    }
10505
10506    /* Hardware requires counter + 1 */
10507    IncrementGcmCounter((byte*)ctr);
10508
10509    ret = wolfSSL_CryptHwMutexLock();
10510    if (ret != 0) {
10511        if (wasAlloc) {
10512            XFREE(authInPadded, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
10513        }
10514        return ret;
10515    }
10516
10517#ifdef WOLFSSL_STM32_CUBEMX
10518    hcryp.Init.pInitVect = (STM_CRYPT_TYPE*)ctr;
10519    hcryp.Init.Header = (STM_CRYPT_TYPE*)authInPadded;
10520
10521#if defined(STM32_HAL_V2)
10522    hcryp.Init.Algorithm = CRYP_AES_GCM;
10523    hcryp.Init.HeaderSize = authPadSz / STM_CRYPT_HEADER_WIDTH;
10524    #ifdef CRYP_KEYIVCONFIG_ONCE
10525    /* allows repeated calls to HAL_CRYP_Decrypt */
10526    hcryp.Init.KeyIVConfigSkip = CRYP_KEYIVCONFIG_ONCE;
10527    #endif
10528    ByteReverseWords(ctr, ctr, WC_AES_BLOCK_SIZE);
10529    hcryp.Init.pInitVect = (STM_CRYPT_TYPE*)ctr;
10530    HAL_CRYP_Init(&hcryp);
10531
10532    #ifndef CRYP_KEYIVCONFIG_ONCE
10533    /* GCM payload phase - can handle partial blocks */
10534    status = HAL_CRYP_Decrypt(&hcryp, (uint32_t*)in,
10535        (blocks * WC_AES_BLOCK_SIZE) + partial, (uint32_t*)out, STM32_HAL_TIMEOUT);
10536    #else
10537    /* GCM payload phase - blocks */
10538    if (blocks) {
10539        status = HAL_CRYP_Decrypt(&hcryp, (uint32_t*)in,
10540            (blocks * WC_AES_BLOCK_SIZE), (uint32_t*)out, STM32_HAL_TIMEOUT);
10541    }
10542    /* GCM payload phase - partial remainder */
10543    if (status == HAL_OK && (partial != 0 || blocks == 0)) {
10544        XMEMSET(partialBlock, 0, sizeof(partialBlock));
10545        XMEMCPY(partialBlock, in + (blocks * WC_AES_BLOCK_SIZE), partial);
10546        status = HAL_CRYP_Decrypt(&hcryp, (uint32_t*)partialBlock, partial,
10547            (uint32_t*)partialBlock, STM32_HAL_TIMEOUT);
10548        XMEMCPY(out + (blocks * WC_AES_BLOCK_SIZE), partialBlock, partial);
10549    }
10550    #endif
10551    if (status == HAL_OK && !tagComputed) {
10552        /* Compute the authTag */
10553        status = HAL_CRYPEx_AESGCM_GenerateAuthTAG(&hcryp, (uint32_t*)tag,
10554            STM32_HAL_TIMEOUT);
10555    }
10556#elif defined(STM32_CRYPTO_AES_ONLY)
10557    /* Set the CRYP parameters */
10558    hcryp.Init.HeaderSize = authPadSz;
10559    if (authPadSz == 0)
10560        hcryp.Init.Header = NULL; /* cannot pass pointer when authIn == 0 */
10561    hcryp.Init.ChainingMode  = CRYP_CHAINMODE_AES_GCM_GMAC;
10562    hcryp.Init.OperatingMode = CRYP_ALGOMODE_DECRYPT;
10563    hcryp.Init.GCMCMACPhase  = CRYP_INIT_PHASE;
10564    HAL_CRYP_Init(&hcryp);
10565
10566    /* GCM init phase */
10567    status = HAL_CRYPEx_AES_Auth(&hcryp, NULL, 0, NULL, STM32_HAL_TIMEOUT);
10568    if (status == HAL_OK) {
10569        /* GCM header phase */
10570        hcryp.Init.GCMCMACPhase = CRYP_HEADER_PHASE;
10571        status = HAL_CRYPEx_AES_Auth(&hcryp, NULL, 0, NULL, STM32_HAL_TIMEOUT);
10572    }
10573    if (status == HAL_OK) {
10574        /* GCM payload phase - blocks */
10575        hcryp.Init.GCMCMACPhase = CRYP_PAYLOAD_PHASE;
10576        if (blocks) {
10577            status = HAL_CRYPEx_AES_Auth(&hcryp, (byte*)in,
10578                (blocks * WC_AES_BLOCK_SIZE), out, STM32_HAL_TIMEOUT);
10579        }
10580    }
10581    if (status == HAL_OK && (partial != 0 || (sz > 0 && blocks == 0))) {
10582        /* GCM payload phase - partial remainder */
10583        XMEMSET(partialBlock, 0, sizeof(partialBlock));
10584        XMEMCPY(partialBlock, in + (blocks * WC_AES_BLOCK_SIZE), partial);
10585        status = HAL_CRYPEx_AES_Auth(&hcryp, (byte*)partialBlock, partial,
10586            (byte*)partialBlock, STM32_HAL_TIMEOUT);
10587        XMEMCPY(out + (blocks * WC_AES_BLOCK_SIZE), partialBlock, partial);
10588    }
10589    if (status == HAL_OK && tagComputed == 0) {
10590        /* GCM final phase */
10591        hcryp.Init.GCMCMACPhase = CRYP_FINAL_PHASE;
10592        status = HAL_CRYPEx_AES_Auth(&hcryp, NULL, sz, (byte*)tag, STM32_HAL_TIMEOUT);
10593    }
10594#else
10595    hcryp.Init.HeaderSize = authPadSz;
10596    HAL_CRYP_Init(&hcryp);
10597    if (blocks) {
10598        /* GCM payload phase - blocks */
10599        status = HAL_CRYPEx_AESGCM_Decrypt(&hcryp, (byte*)in,
10600            (blocks * WC_AES_BLOCK_SIZE), out, STM32_HAL_TIMEOUT);
10601    }
10602    if (status == HAL_OK && (partial != 0 || blocks == 0)) {
10603        /* GCM payload phase - partial remainder */
10604        XMEMSET(partialBlock, 0, sizeof(partialBlock));
10605        XMEMCPY(partialBlock, in + (blocks * WC_AES_BLOCK_SIZE), partial);
10606        status = HAL_CRYPEx_AESGCM_Decrypt(&hcryp, (byte*)partialBlock, partial,
10607            (byte*)partialBlock, STM32_HAL_TIMEOUT);
10608        XMEMCPY(out + (blocks * WC_AES_BLOCK_SIZE), partialBlock, partial);
10609    }
10610    if (status == HAL_OK && tagComputed == 0) {
10611        /* Compute the authTag */
10612        status = HAL_CRYPEx_AESGCM_Finish(&hcryp, sz, (byte*)tag, STM32_HAL_TIMEOUT);
10613    }
10614#endif
10615
10616    if (status != HAL_OK)
10617        ret = AES_GCM_AUTH_E;
10618
10619    HAL_CRYP_DeInit(&hcryp);
10620
10621#else /* Standard Peripheral Library */
10622    ByteReverseWords(keyCopy, (word32*)aes->key, aes->keylen);
10623
10624    /* Input size and auth size need to be the actual sizes, even though
10625     * they are not block aligned, because this length (in bits) is used
10626     * in the final GHASH. */
10627    XMEMSET(partialBlock, 0, sizeof(partialBlock)); /* use this to get tag */
10628    status = CRYP_AES_GCM(MODE_DECRYPT, (uint8_t*)ctr,
10629                         (uint8_t*)keyCopy,      keySize * 8,
10630                         (uint8_t*)in,           sz,
10631                         (uint8_t*)authInPadded, authInSz,
10632                         (uint8_t*)out,          (uint8_t*)partialBlock);
10633    if (status != SUCCESS)
10634        ret = AES_GCM_AUTH_E;
10635    if (tagComputed == 0)
10636        XMEMCPY(tag, partialBlock, authTagSz);
10637#endif /* WOLFSSL_STM32_CUBEMX */
10638    wolfSSL_CryptHwMutexUnLock();
10639    wc_Stm32_Aes_Cleanup();
10640
10641    /* Check authentication tag */
10642    if (ConstantCompare((const byte*)tagExpected, (byte*)tag, authTagSz) != 0) {
10643        ret = AES_GCM_AUTH_E;
10644    }
10645
10646    /* Free memory */
10647    if (wasAlloc) {
10648        XFREE(authInPadded, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
10649    }
10650
10651    return ret;
10652}
10653
10654#endif /* STM32_CRYPTO_AES_GCM */
10655
10656#if !defined(WOLFSSL_ARMASM)
10657#ifdef WOLFSSL_AESNI
10658/* For performance reasons, this code needs to be not inlined. */
10659int WARN_UNUSED_RESULT AES_GCM_decrypt_C(
10660                      Aes* aes, byte* out, const byte* in, word32 sz,
10661                      const byte* iv, word32 ivSz,
10662                      const byte* authTag, word32 authTagSz,
10663                      const byte* authIn, word32 authInSz);
10664#else
10665static
10666#endif
10667int WARN_UNUSED_RESULT AES_GCM_decrypt_C(
10668                      Aes* aes, byte* out, const byte* in, word32 sz,
10669                      const byte* iv, word32 ivSz,
10670                      const byte* authTag, word32 authTagSz,
10671                      const byte* authIn, word32 authInSz)
10672{
10673    int ret;
10674    word32 blocks = sz / WC_AES_BLOCK_SIZE;
10675    word32 partial = sz % WC_AES_BLOCK_SIZE;
10676    const byte* c = in;
10677    byte* p = out;
10678    ALIGN16 byte counter[WC_AES_BLOCK_SIZE];
10679    ALIGN16 byte scratch[WC_AES_BLOCK_SIZE];
10680    ALIGN16 byte Tprime[WC_AES_BLOCK_SIZE];
10681    ALIGN16 byte EKY0[WC_AES_BLOCK_SIZE];
10682    volatile sword32 res;
10683
10684    if (ivSz == GCM_NONCE_MID_SZ) {
10685        /* Counter is IV with bottom 4 bytes set to: 0x00,0x00,0x00,0x01. */
10686        XMEMCPY(counter, iv, ivSz);
10687        XMEMSET(counter + GCM_NONCE_MID_SZ, 0,
10688                                         WC_AES_BLOCK_SIZE - GCM_NONCE_MID_SZ - 1);
10689        counter[WC_AES_BLOCK_SIZE - 1] = 1;
10690    }
10691    else {
10692        /* Counter is GHASH of IV. */
10693#ifdef OPENSSL_EXTRA
10694        word32 aadTemp = aes->gcm.aadLen;
10695        aes->gcm.aadLen = 0;
10696#endif
10697        GHASH(&aes->gcm, NULL, 0, iv, ivSz, counter, WC_AES_BLOCK_SIZE);
10698#ifdef OPENSSL_EXTRA
10699        aes->gcm.aadLen = aadTemp;
10700#endif
10701    }
10702
10703    /* Calc the authTag again using received auth data and the cipher text */
10704    GHASH(&aes->gcm, authIn, authInSz, in, sz, Tprime, sizeof(Tprime));
10705    ret = wc_AesEncrypt(aes, counter, EKY0);
10706    if (ret != 0)
10707        return ret;
10708    xorbuf(Tprime, EKY0, sizeof(Tprime));
10709#ifdef WC_AES_GCM_DEC_AUTH_EARLY
10710    /* ConstantCompare returns the cumulative bitwise or of the bitwise xor of
10711     * the pairwise bytes in the strings.
10712     */
10713    res = ConstantCompare(authTag, Tprime, authTagSz);
10714    /* convert positive retval from ConstantCompare() to all-1s word, in
10715     * constant time.
10716     */
10717    res = 0 - (sword32)(((word32)(0 - res)) >> 31U);
10718    ret = res & AES_GCM_AUTH_E;
10719    if (ret != 0)
10720        return ret;
10721#endif
10722
10723#ifdef OPENSSL_EXTRA
10724    if (!out) {
10725        /* authenticated, non-confidential data */
10726        /* store AAD size for next call */
10727        aes->gcm.aadLen = authInSz;
10728    }
10729#endif
10730
10731#if defined(WOLFSSL_PIC32MZ_CRYPT)
10732    if (blocks) {
10733        /* use initial IV for HW, but don't use it below */
10734        XMEMCPY(aes->reg, counter, WC_AES_BLOCK_SIZE);
10735
10736        ret = wc_Pic32AesCrypt(
10737            aes->key, aes->keylen, aes->reg, WC_AES_BLOCK_SIZE,
10738            out, in, (blocks * WC_AES_BLOCK_SIZE),
10739            PIC32_DECRYPTION, PIC32_ALGO_AES, PIC32_CRYPTOALGO_AES_GCM);
10740        if (ret != 0)
10741            return ret;
10742    }
10743    /* process remainder using partial handling */
10744#endif
10745
10746#if defined(HAVE_AES_ECB) && !defined(WOLFSSL_PIC32MZ_CRYPT)
10747    /* some hardware acceleration can gain performance from doing AES encryption
10748     * of the whole buffer at once */
10749    if (c != p && blocks > 0) { /* can not handle inline decryption */
10750        while (blocks--) {
10751            IncrementGcmCounter(counter);
10752            XMEMCPY(p, counter, WC_AES_BLOCK_SIZE);
10753            p += WC_AES_BLOCK_SIZE;
10754        }
10755
10756        /* reset number of blocks and then do encryption */
10757        blocks = sz / WC_AES_BLOCK_SIZE;
10758
10759        wc_AesEcbEncrypt(aes, out, out, WC_AES_BLOCK_SIZE * blocks);
10760        xorbuf(out, c, WC_AES_BLOCK_SIZE * blocks);
10761        c += WC_AES_BLOCK_SIZE * blocks;
10762    }
10763    else
10764#endif /* HAVE_AES_ECB && !PIC32MZ */
10765    {
10766        while (blocks--) {
10767            IncrementGcmCounter(counter);
10768        #if !defined(WOLFSSL_PIC32MZ_CRYPT)
10769            ret = wc_AesEncrypt(aes, counter, scratch);
10770            if (ret != 0)
10771                return ret;
10772            xorbufout(p, scratch, c, WC_AES_BLOCK_SIZE);
10773        #endif
10774            p += WC_AES_BLOCK_SIZE;
10775            c += WC_AES_BLOCK_SIZE;
10776        }
10777    }
10778
10779    if (partial != 0) {
10780        IncrementGcmCounter(counter);
10781        ret = wc_AesEncrypt(aes, counter, scratch);
10782        if (ret != 0)
10783            return ret;
10784        xorbuf(scratch, c, partial);
10785        XMEMCPY(p, scratch, partial);
10786    }
10787
10788#ifndef WC_AES_GCM_DEC_AUTH_EARLY
10789    /* ConstantCompare returns the cumulative bitwise or of the bitwise xor of
10790     * the pairwise bytes in the strings.
10791     */
10792    res = ConstantCompare(authTag, Tprime, (int)authTagSz);
10793    /* convert positive retval from ConstantCompare() to all-1s word, in
10794     * constant time.
10795     */
10796    res = 0 - (sword32)(((word32)(0 - res)) >> 31U);
10797    /* now use res as a mask for constant time return of ret, unless tag
10798     * mismatch, whereupon AES_GCM_AUTH_E is returned.
10799     */
10800    ret = (ret & ~res);
10801    ret |= (res & WC_NO_ERR_TRACE(AES_GCM_AUTH_E));
10802#endif
10803    return ret;
10804}
10805#elif defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
10806static int AES_GCM_decrypt_ARM(Aes* aes, byte* out, const byte* in,
10807    word32 sz, const byte* iv, word32 ivSz, const byte* authTag,
10808    word32 authTagSz, const byte* authIn, word32 authInSz)
10809{
10810    word32 blocks;
10811    word32 partial;
10812    byte counter[WC_AES_BLOCK_SIZE];
10813    byte initialCounter[WC_AES_BLOCK_SIZE];
10814    byte scratch[WC_AES_BLOCK_SIZE];
10815    byte x[WC_AES_BLOCK_SIZE];
10816
10817    XMEMSET(initialCounter, 0, WC_AES_BLOCK_SIZE);
10818    if (ivSz == GCM_NONCE_MID_SZ) {
10819        XMEMCPY(initialCounter, iv, ivSz);
10820        initialCounter[WC_AES_BLOCK_SIZE - 1] = 1;
10821    }
10822    else {
10823        GHASH(&aes->gcm, NULL, 0, iv, ivSz, initialCounter, WC_AES_BLOCK_SIZE);
10824    }
10825    XMEMCPY(counter, initialCounter, WC_AES_BLOCK_SIZE);
10826
10827    XMEMSET(x, 0, WC_AES_BLOCK_SIZE);
10828    /* Hash in the Additional Authentication Data */
10829    if (authInSz != 0 && authIn != NULL) {
10830        blocks = authInSz / WC_AES_BLOCK_SIZE;
10831        partial = authInSz % WC_AES_BLOCK_SIZE;
10832        if (blocks > 0) {
10833            GCM_GMULT_LEN(&aes->gcm, x, authIn, blocks * WC_AES_BLOCK_SIZE);
10834            authIn += blocks * WC_AES_BLOCK_SIZE;
10835        }
10836        if (partial != 0) {
10837            XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
10838            XMEMCPY(scratch, authIn, partial);
10839            GCM_GMULT_LEN(&aes->gcm, x, scratch, WC_AES_BLOCK_SIZE);
10840        }
10841    }
10842
10843    blocks = sz / WC_AES_BLOCK_SIZE;
10844    partial = sz % WC_AES_BLOCK_SIZE;
10845    /* do as many blocks as possible */
10846    if (blocks > 0) {
10847        GCM_GMULT_LEN(&aes->gcm, x, in, blocks * WC_AES_BLOCK_SIZE);
10848
10849    #if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON)
10850    #ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
10851        if (sz >= 32)
10852    #endif
10853        {
10854            AES_GCM_encrypt_NEON(in, out, blocks * WC_AES_BLOCK_SIZE,
10855                (const unsigned char*)aes->key, aes->rounds, counter);
10856        }
10857    #ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
10858        else
10859    #endif
10860    #endif
10861    #ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
10862        {
10863            AES_GCM_encrypt(in, out, blocks * WC_AES_BLOCK_SIZE,
10864                (const unsigned char*)aes->key, aes->rounds, counter);
10865        }
10866    #endif
10867        in += blocks * WC_AES_BLOCK_SIZE;
10868        out += blocks * WC_AES_BLOCK_SIZE;
10869    }
10870    if (partial != 0) {
10871        XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
10872        XMEMCPY(scratch, in, partial);
10873        GCM_GMULT_LEN(&aes->gcm, x, scratch, WC_AES_BLOCK_SIZE);
10874
10875    #if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON) && \
10876        defined(WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP)
10877        {
10878            AES_GCM_encrypt_NEON(in, scratch, WC_AES_BLOCK_SIZE,
10879                (const unsigned char*)aes->key, aes->rounds, counter);
10880        }
10881    #else
10882        {
10883            AES_GCM_encrypt(in, scratch, WC_AES_BLOCK_SIZE,
10884                (const unsigned char*)aes->key, aes->rounds, counter);
10885        }
10886    #endif
10887        XMEMCPY(out, scratch, partial);
10888    }
10889
10890    XMEMSET(scratch, 0, WC_AES_BLOCK_SIZE);
10891    FlattenSzInBits(&scratch[0], authInSz);
10892    FlattenSzInBits(&scratch[8], sz);
10893    GCM_GMULT_LEN(&aes->gcm, x, scratch, WC_AES_BLOCK_SIZE);
10894#if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON) && \
10895    defined(WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP)
10896    {
10897        AES_ECB_encrypt_NEON(initialCounter, scratch, WC_AES_BLOCK_SIZE,
10898            (const unsigned char*)aes->key, aes->rounds);
10899    }
10900#else
10901    {
10902        AES_ECB_encrypt(initialCounter, scratch, WC_AES_BLOCK_SIZE,
10903            (const unsigned char*)aes->key, aes->rounds);
10904    }
10905#endif
10906    xorbuf(x, scratch, authTagSz);
10907    if (authTag != NULL) {
10908        if (ConstantCompare(authTag, x, authTagSz) != 0) {
10909            return AES_GCM_AUTH_E;
10910        }
10911    }
10912
10913    return 0;
10914}
10915#endif
10916
10917/* Software AES - GCM Decrypt */
10918int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
10919                     const byte* iv, word32 ivSz,
10920                     const byte* authTag, word32 authTagSz,
10921                     const byte* authIn, word32 authInSz)
10922{
10923    int ret;
10924#ifdef WOLFSSL_AESNI
10925    int res = WC_NO_ERR_TRACE(AES_GCM_AUTH_E);
10926#endif
10927
10928    /* argument checks */
10929    /* If the sz is non-zero, both in and out must be set. If sz is 0,
10930     * in and out are don't cares, as this is is the GMAC case. */
10931    if (aes == NULL || iv == NULL || (sz != 0 && (in == NULL || out == NULL)) ||
10932        authTag == NULL || authTagSz > WC_AES_BLOCK_SIZE ||
10933        authTagSz < WOLFSSL_MIN_AUTH_TAG_SZ || ivSz == 0) {
10934
10935        return BAD_FUNC_ARG;
10936    }
10937
10938#ifdef WOLF_CRYPTO_CB
10939    #ifndef WOLF_CRYPTO_CB_FIND
10940    if (aes->devId != INVALID_DEVID)
10941    #endif
10942    {
10943        int crypto_cb_ret =
10944            wc_CryptoCb_AesGcmDecrypt(aes, out, in, sz, iv, ivSz,
10945                                      authTag, authTagSz, authIn, authInSz);
10946        if (crypto_cb_ret != WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE))
10947            return crypto_cb_ret;
10948        /* fall-through when unavailable */
10949    }
10950#endif
10951
10952#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)
10953    /* if async and byte count above threshold */
10954    /* only 12-byte IV is supported in HW */
10955    if (aes->asyncDev.marker == WOLFSSL_ASYNC_MARKER_AES &&
10956                    sz >= WC_ASYNC_THRESH_AES_GCM && ivSz == GCM_NONCE_MID_SZ) {
10957    #if defined(HAVE_CAVIUM)
10958        #ifdef HAVE_CAVIUM_V
10959        if (authInSz == 20) { /* Nitrox V GCM is only working with 20 byte AAD */
10960            return NitroxAesGcmDecrypt(aes, out, in, sz,
10961                (const byte*)aes->devKey, aes->keylen, iv, ivSz,
10962                authTag, authTagSz, authIn, authInSz);
10963        }
10964        #endif
10965    #elif defined(HAVE_INTEL_QA)
10966        return IntelQaSymAesGcmDecrypt(&aes->asyncDev, out, in, sz,
10967            (const byte*)aes->devKey, aes->keylen, iv, ivSz,
10968            authTag, authTagSz, authIn, authInSz);
10969    #elif defined(WOLFSSL_ASYNC_CRYPT_SW)
10970        if (wc_AsyncSwInit(&aes->asyncDev, ASYNC_SW_AES_GCM_DECRYPT)) {
10971            WC_ASYNC_SW* sw = &aes->asyncDev.sw;
10972            sw->aes.aes = aes;
10973            sw->aes.out = out;
10974            sw->aes.in = in;
10975            sw->aes.sz = sz;
10976            sw->aes.iv = iv;
10977            sw->aes.ivSz = ivSz;
10978            sw->aes.authTag = (byte*)authTag;
10979            sw->aes.authTagSz = authTagSz;
10980            sw->aes.authIn = authIn;
10981            sw->aes.authInSz = authInSz;
10982            return WC_PENDING_E;
10983        }
10984    #endif
10985    }
10986#endif /* WOLFSSL_ASYNC_CRYPT */
10987
10988#ifdef WOLFSSL_SILABS_SE_ACCEL
10989    return wc_AesGcmDecrypt_silabs(
10990        aes, out, in, sz, iv, ivSz,
10991        authTag, authTagSz, authIn, authInSz);
10992
10993#endif
10994#if defined(WOLFSSL_MICROCHIP_TA100) && defined(WOLFSSL_MICROCHIP_AESGCM)
10995#ifndef TA_AES_GCM_MAX_DATA_SIZE
10996    #define TA_AES_GCM_MAX_DATA_SIZE 996u
10997#endif
10998    if (aes != NULL &&
10999        aes->keylen == TA_KEY_TYPE_AES128_SIZE &&
11000        ivSz == TA_AES_GCM_IV_LENGTH &&
11001        authTagSz == TA_AES_GCM_TAG_LENGTH &&
11002        sz <= TA_AES_GCM_MAX_DATA_SIZE &&
11003        authInSz <= (word32)(TA_AES_GCM_MAX_DATA_SIZE - sz)) {
11004        return wc_Microchip_AesGcmDecrypt(
11005            aes, out, in, sz, iv, ivSz,
11006            authTag, authTagSz, authIn, authInSz);
11007    }
11008#endif
11009
11010#ifdef STM32_CRYPTO_AES_GCM
11011    /* The STM standard peripheral library API's doesn't support partial blocks */
11012    return wc_AesGcmDecrypt_STM32(
11013        aes, out, in, sz, iv, ivSz,
11014        authTag, authTagSz, authIn, authInSz);
11015#endif /* STM32_CRYPTO_AES_GCM */
11016
11017#if defined(WOLFSSL_PSOC6_CRYPTO)
11018    return wc_Psoc6_Aes_GcmDecrypt(aes, out, in, sz, iv, ivSz, authTag,
11019                                   authTagSz, authIn, authInSz);
11020#endif /* WOLFSSL_PSOC6_CRYPTO */
11021
11022    VECTOR_REGISTERS_PUSH;
11023
11024#if defined(WOLFSSL_ARMASM)
11025#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
11026#ifndef __aarch64__
11027    {
11028    #ifdef OPENSSL_EXTRA
11029        word32 reg[WC_AES_BLOCK_SIZE / sizeof(word32)];
11030        XMEMCPY(reg, aes->reg, sizeof(reg));
11031    #endif
11032        ret = AES_GCM_decrypt_AARCH32(in, out, sz, iv, ivSz, authTag, authTagSz,
11033            authIn, authInSz, (byte*)aes->key, aes->gcm.H, (byte*)aes->tmp,
11034            (byte*)aes->reg, aes->rounds);
11035    #ifdef OPENSSL_EXTRA
11036        XMEMCPY(aes->reg, reg, sizeof(reg));
11037    #endif
11038    }
11039#else
11040    if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) {
11041    #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
11042        if (aes->use_sha3_hw_crypto) {
11043            ret = AES_GCM_decrypt_AARCH64_EOR3(in, out, sz, iv, ivSz, authTag,
11044                authTagSz, authIn, authInSz, (byte*)aes->key, aes->gcm.H,
11045                (byte*)aes->tmp, (byte*)aes->reg, aes->rounds);
11046        }
11047        else
11048    #endif
11049        {
11050            ret = AES_GCM_decrypt_AARCH64(in, out, sz, iv, ivSz, authTag,
11051                authTagSz, authIn, authInSz, (byte*)aes->key, aes->gcm.H,
11052                (byte*)aes->tmp, (byte*)aes->reg, aes->rounds);
11053        }
11054    }
11055    else
11056#endif /* !__aarch64__ */
11057#endif /* !WOLFSSL_ARMASM_NO_HW_CRYPTO */
11058#if defined(__aarch64__) || defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
11059    {
11060        ret = AES_GCM_decrypt_ARM(aes, out, in, sz, iv, ivSz, authTag,
11061            authTagSz, authIn, authInSz);
11062    }
11063#endif /* __aarch64__ || WOLFSSL_ARMASM_NO_HW_CRYPTO */
11064#else
11065#ifdef WOLFSSL_AESNI
11066    if (aes->use_aesni) {
11067#ifdef HAVE_INTEL_AVX2
11068        if (IS_INTEL_AVX2(intel_flags)) {
11069            AES_GCM_decrypt_avx2(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
11070                                 authTagSz, (byte*)aes->key, (int)aes->rounds, &res);
11071            if (res == 0)
11072                ret = AES_GCM_AUTH_E;
11073            else
11074                ret = 0;
11075        }
11076        else
11077#endif
11078#if defined(HAVE_INTEL_AVX1)
11079        if (IS_INTEL_AVX1(intel_flags)) {
11080            AES_GCM_decrypt_avx1(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
11081                                 authTagSz, (byte*)aes->key, (int)aes->rounds, &res);
11082            if (res == 0)
11083                ret = AES_GCM_AUTH_E;
11084            else
11085                ret = 0;
11086        }
11087        else
11088#endif
11089        {
11090            AES_GCM_decrypt_aesni(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
11091                            authTagSz, (byte*)aes->key, (int)aes->rounds, &res);
11092            if (res == 0)
11093                ret = AES_GCM_AUTH_E;
11094            else
11095                ret = 0;
11096        }
11097    }
11098    else
11099#endif /* WOLFSSL_AESNI */
11100    {
11101        ret = AES_GCM_decrypt_C(aes, out, in, sz, iv, ivSz, authTag, authTagSz,
11102                                                             authIn, authInSz);
11103    }
11104#endif
11105
11106    VECTOR_REGISTERS_POP;
11107
11108    return ret;
11109}
11110#endif
11111#endif /* HAVE_AES_DECRYPT || HAVE_AESGCM_DECRYPT */
11112
11113#ifdef WOLFSSL_AESGCM_STREAM
11114
11115/* Initialize the AES GCM cipher with an IV. C implementation.
11116 *
11117 * @param [in, out] aes   AES object.
11118 * @param [in]      iv    IV/nonce buffer.
11119 * @param [in]      ivSz  Length of IV/nonce data.
11120 */
11121static WARN_UNUSED_RESULT int AesGcmInit_C(Aes* aes, const byte* iv, word32 ivSz)
11122{
11123    ALIGN32 byte counter[WC_AES_BLOCK_SIZE];
11124    int ret;
11125
11126    if (ivSz == GCM_NONCE_MID_SZ) {
11127        /* Counter is IV with bottom 4 bytes set to: 0x00,0x00,0x00,0x01. */
11128        XMEMCPY(counter, iv, ivSz);
11129        XMEMSET(counter + GCM_NONCE_MID_SZ, 0,
11130                                         WC_AES_BLOCK_SIZE - GCM_NONCE_MID_SZ - 1);
11131        counter[WC_AES_BLOCK_SIZE - 1] = 1;
11132    }
11133    else {
11134        /* Counter is GHASH of IV. */
11135    #ifdef OPENSSL_EXTRA
11136        word32 aadTemp = aes->gcm.aadLen;
11137        aes->gcm.aadLen = 0;
11138    #endif
11139        GHASH(&aes->gcm, NULL, 0, iv, ivSz, counter, WC_AES_BLOCK_SIZE);
11140    #ifdef OPENSSL_EXTRA
11141        aes->gcm.aadLen = aadTemp;
11142    #endif
11143    }
11144
11145    /* Copy in the counter for use with cipher. */
11146    XMEMCPY(AES_COUNTER(aes), counter, WC_AES_BLOCK_SIZE);
11147    /* Encrypt initial counter into a buffer for GCM. */
11148    ret = wc_AesEncrypt(aes, counter, AES_INITCTR(aes));
11149    if (ret != 0)
11150        return ret;
11151    /* Reset state fields. */
11152    aes->over = 0;
11153    aes->aSz = 0;
11154    aes->cSz = 0;
11155    /* Initialization for GHASH. */
11156    GHASH_INIT(aes);
11157
11158    return 0;
11159}
11160
11161/* Update the AES GCM cipher with data. C implementation.
11162 *
11163 * Only enciphers data.
11164 *
11165 * @param [in, out] aes  AES object.
11166 * @param [in]      out  Cipher text or plaintext buffer.
11167 * @param [in]      in   Plaintext or cipher text buffer.
11168 * @param [in]      sz   Length of data.
11169 */
11170static WARN_UNUSED_RESULT int AesGcmCryptUpdate_C(
11171    Aes* aes, byte* out, const byte* in, word32 sz)
11172{
11173    word32 blocks;
11174    word32 partial;
11175    int ret;
11176
11177    /* Check if previous encrypted block was not used up. */
11178    if (aes->over > 0) {
11179        byte pSz = (byte)(WC_AES_BLOCK_SIZE - aes->over);
11180        if (pSz > sz) pSz = (byte)sz;
11181
11182        /* Use some/all of last encrypted block. */
11183        xorbufout(out, AES_LASTBLOCK(aes) + aes->over, in, pSz);
11184        aes->over = (aes->over + pSz) & (WC_AES_BLOCK_SIZE - 1);
11185
11186        /* Some data used. */
11187        sz  -= pSz;
11188        in  += pSz;
11189        out += pSz;
11190    }
11191
11192    /* Calculate the number of blocks needing to be encrypted and any leftover.
11193     */
11194    blocks  = sz / WC_AES_BLOCK_SIZE;
11195    partial = sz & (WC_AES_BLOCK_SIZE - 1);
11196
11197#if defined(HAVE_AES_ECB)
11198    /* Some hardware acceleration can gain performance from doing AES encryption
11199     * of the whole buffer at once.
11200     * Overwrites the cipher text before using plaintext - no inline encryption.
11201     */
11202    if ((out != in) && blocks > 0) {
11203        word32 b;
11204        /* Place incrementing counter blocks into cipher text. */
11205        for (b = 0; b < blocks; b++) {
11206            IncrementGcmCounter(AES_COUNTER(aes));
11207            XMEMCPY(out + b * WC_AES_BLOCK_SIZE, AES_COUNTER(aes), WC_AES_BLOCK_SIZE);
11208        }
11209
11210        /* Encrypt counter blocks. */
11211        wc_AesEcbEncrypt(aes, out, out, WC_AES_BLOCK_SIZE * blocks);
11212        /* XOR in plaintext. */
11213        xorbuf(out, in, WC_AES_BLOCK_SIZE * blocks);
11214        /* Skip over processed data. */
11215        in += WC_AES_BLOCK_SIZE * blocks;
11216        out += WC_AES_BLOCK_SIZE * blocks;
11217    }
11218    else
11219#endif /* HAVE_AES_ECB */
11220    {
11221        /* Encrypt block by block. */
11222        while (blocks--) {
11223            ALIGN32 byte scratch[WC_AES_BLOCK_SIZE];
11224            IncrementGcmCounter(AES_COUNTER(aes));
11225            /* Encrypt counter into a buffer. */
11226            ret = wc_AesEncrypt(aes, AES_COUNTER(aes), scratch);
11227            if (ret != 0)
11228                return ret;
11229            /* XOR plain text into encrypted counter into cipher text buffer. */
11230            xorbufout(out, scratch, in, WC_AES_BLOCK_SIZE);
11231            /* Data complete. */
11232            in  += WC_AES_BLOCK_SIZE;
11233            out += WC_AES_BLOCK_SIZE;
11234        }
11235    }
11236
11237    if (partial != 0) {
11238        /* Generate an extra block and use up as much as needed. */
11239        IncrementGcmCounter(AES_COUNTER(aes));
11240        /* Encrypt counter into cache. */
11241        ret = wc_AesEncrypt(aes, AES_COUNTER(aes), AES_LASTBLOCK(aes));
11242        if (ret != 0)
11243            return ret;
11244        /* XOR plain text into encrypted counter into cipher text buffer. */
11245        xorbufout(out, AES_LASTBLOCK(aes), in, partial);
11246        /* Keep amount of encrypted block used. */
11247        aes->over = (byte)partial;
11248    }
11249
11250    return 0;
11251}
11252
11253/* Calculates authentication tag for AES GCM. C implementation.
11254 *
11255 * @param [in, out] aes        AES object.
11256 * @param [out]     authTag    Buffer to store authentication tag in.
11257 * @param [in]      authTagSz  Length of tag to create.
11258 */
11259static WARN_UNUSED_RESULT int AesGcmFinal_C(
11260    Aes* aes, byte* authTag, word32 authTagSz)
11261{
11262    /* Calculate authentication tag. */
11263    GHASH_FINAL(aes, authTag, authTagSz);
11264    /* XOR in as much of encrypted counter as is required. */
11265    xorbuf(authTag, AES_INITCTR(aes), authTagSz);
11266#ifdef OPENSSL_EXTRA
11267    /* store AAD size for next call */
11268    aes->gcm.aadLen = aes->aSz;
11269#endif
11270    /* Zeroize last block to protect sensitive data. */
11271    ForceZero(AES_LASTBLOCK(aes), WC_AES_BLOCK_SIZE);
11272
11273    return 0;
11274}
11275
11276#ifdef WOLFSSL_AESNI
11277
11278#ifdef __cplusplus
11279    extern "C" {
11280#endif
11281
11282/* Assembly code implementations in: aes_gcm_asm.S */
11283#ifdef HAVE_INTEL_AVX2
11284extern void AES_GCM_init_avx2(const unsigned char* key, int nr,
11285    const unsigned char* ivec, unsigned int ibytes, unsigned char* h,
11286    unsigned char* counter, unsigned char* initCtr);
11287extern void AES_GCM_aad_update_avx2(const unsigned char* addt,
11288    unsigned int abytes, unsigned char* tag, unsigned char* h);
11289extern void AES_GCM_encrypt_block_avx2(const unsigned char* key, int nr,
11290    unsigned char* out, const unsigned char* in, unsigned char* counter);
11291extern void AES_GCM_ghash_block_avx2(const unsigned char* data,
11292    unsigned char* tag, unsigned char* h);
11293
11294extern void AES_GCM_encrypt_update_avx2(const unsigned char* key, int nr,
11295    unsigned char* out, const unsigned char* in, unsigned int nbytes,
11296    unsigned char* tag, unsigned char* h, unsigned char* counter);
11297extern void AES_GCM_encrypt_final_avx2(unsigned char* tag,
11298    unsigned char* authTag, unsigned int tbytes, unsigned int nbytes,
11299    unsigned int abytes, unsigned char* h, unsigned char* initCtr);
11300#endif
11301#ifdef HAVE_INTEL_AVX1
11302extern void AES_GCM_init_avx1(const unsigned char* key, int nr,
11303    const unsigned char* ivec, unsigned int ibytes, unsigned char* h,
11304    unsigned char* counter, unsigned char* initCtr);
11305extern void AES_GCM_aad_update_avx1(const unsigned char* addt,
11306    unsigned int abytes, unsigned char* tag, unsigned char* h);
11307extern void AES_GCM_encrypt_block_avx1(const unsigned char* key, int nr,
11308    unsigned char* out, const unsigned char* in, unsigned char* counter);
11309extern void AES_GCM_ghash_block_avx1(const unsigned char* data,
11310    unsigned char* tag, unsigned char* h);
11311
11312extern void AES_GCM_encrypt_update_avx1(const unsigned char* key, int nr,
11313    unsigned char* out, const unsigned char* in, unsigned int nbytes,
11314    unsigned char* tag, unsigned char* h, unsigned char* counter);
11315extern void AES_GCM_encrypt_final_avx1(unsigned char* tag,
11316    unsigned char* authTag, unsigned int tbytes, unsigned int nbytes,
11317    unsigned int abytes, unsigned char* h, unsigned char* initCtr);
11318#endif
11319extern void AES_GCM_init_aesni(const unsigned char* key, int nr,
11320    const unsigned char* ivec, unsigned int ibytes, unsigned char* h,
11321    unsigned char* counter, unsigned char* initCtr);
11322extern void AES_GCM_aad_update_aesni(const unsigned char* addt,
11323    unsigned int abytes, unsigned char* tag, unsigned char* h);
11324extern void AES_GCM_encrypt_block_aesni(const unsigned char* key, int nr,
11325    unsigned char* out, const unsigned char* in, unsigned char* counter);
11326extern void AES_GCM_ghash_block_aesni(const unsigned char* data,
11327    unsigned char* tag, unsigned char* h);
11328
11329extern void AES_GCM_encrypt_update_aesni(const unsigned char* key, int nr,
11330    unsigned char* out, const unsigned char* in, unsigned int nbytes,
11331    unsigned char* tag, unsigned char* h, unsigned char* counter);
11332extern void AES_GCM_encrypt_final_aesni(unsigned char* tag,
11333    unsigned char* authTag, unsigned int tbytes, unsigned int nbytes,
11334    unsigned int abytes, unsigned char* h, unsigned char* initCtr);
11335
11336#ifdef __cplusplus
11337    } /* extern "C" */
11338#endif
11339
11340/* Initialize the AES GCM cipher with an IV. AES-NI implementations.
11341 *
11342 * @param [in, out] aes   AES object.
11343 * @param [in]      iv    IV/nonce buffer.
11344 * @param [in]      ivSz  Length of IV/nonce data.
11345 */
11346static WARN_UNUSED_RESULT int AesGcmInit_aesni(
11347    Aes* aes, const byte* iv, word32 ivSz)
11348{
11349    ASSERT_SAVED_VECTOR_REGISTERS();
11350
11351    /* Reset state fields. */
11352    aes->over = 0;
11353    aes->aSz = 0;
11354    aes->cSz = 0;
11355    /* Set tag to all zeros as initial value. */
11356    XMEMSET(AES_TAG(aes), 0, WC_AES_BLOCK_SIZE);
11357    /* Reset counts of AAD and cipher text. */
11358    aes->aOver = 0;
11359    aes->cOver = 0;
11360
11361#ifdef HAVE_INTEL_AVX2
11362    if (IS_INTEL_AVX2(intel_flags)) {
11363        AES_GCM_init_avx2((byte*)aes->key, (int)aes->rounds, iv, ivSz,
11364            aes->gcm.H, AES_COUNTER(aes), AES_INITCTR(aes));
11365    }
11366    else
11367#endif
11368#ifdef HAVE_INTEL_AVX1
11369    if (IS_INTEL_AVX1(intel_flags)) {
11370        AES_GCM_init_avx1((byte*)aes->key, (int)aes->rounds, iv, ivSz,
11371            aes->gcm.H, AES_COUNTER(aes), AES_INITCTR(aes));
11372    }
11373    else
11374#endif
11375    {
11376        AES_GCM_init_aesni((byte*)aes->key, (int)aes->rounds, iv, ivSz,
11377            aes->gcm.H, AES_COUNTER(aes), AES_INITCTR(aes));
11378    }
11379
11380    return 0;
11381}
11382
11383/* Update the AES GCM for encryption with authentication data.
11384 *
11385 * Implementation uses AVX2, AVX1 or straight AES-NI optimized assembly code.
11386 *
11387 * @param [in, out] aes   AES object.
11388 * @param [in]      a     Buffer holding authentication data.
11389 * @param [in]      aSz   Length of authentication data in bytes.
11390 * @param [in]      endA  Whether no more authentication data is expected.
11391 */
11392static WARN_UNUSED_RESULT int AesGcmAadUpdate_aesni(
11393    Aes* aes, const byte* a, word32 aSz, int endA)
11394{
11395    word32 blocks;
11396    int partial;
11397
11398    ASSERT_SAVED_VECTOR_REGISTERS();
11399
11400    if (aSz != 0 && a != NULL) {
11401        /* Total count of AAD updated. */
11402        aes->aSz += aSz;
11403        /* Check if we have unprocessed data. */
11404        if (aes->aOver > 0) {
11405            /* Calculate amount we can use - fill up the block. */
11406            byte sz = (byte)(WC_AES_BLOCK_SIZE - aes->aOver);
11407            if (sz > aSz) {
11408                sz = (byte)aSz;
11409            }
11410            /* Copy extra into last GHASH block array and update count. */
11411            XMEMCPY(AES_LASTGBLOCK(aes) + aes->aOver, a, sz);
11412            aes->aOver = (byte)(aes->aOver + sz);
11413            if (aes->aOver == WC_AES_BLOCK_SIZE) {
11414                /* We have filled up the block and can process. */
11415            #ifdef HAVE_INTEL_AVX2
11416                if (IS_INTEL_AVX2(intel_flags)) {
11417                    AES_GCM_ghash_block_avx2(AES_LASTGBLOCK(aes), AES_TAG(aes),
11418                                             aes->gcm.H);
11419                }
11420                else
11421            #endif
11422            #ifdef HAVE_INTEL_AVX1
11423                if (IS_INTEL_AVX1(intel_flags)) {
11424                    AES_GCM_ghash_block_avx1(AES_LASTGBLOCK(aes), AES_TAG(aes),
11425                                             aes->gcm.H);
11426                }
11427                else
11428            #endif
11429                {
11430                    AES_GCM_ghash_block_aesni(AES_LASTGBLOCK(aes), AES_TAG(aes),
11431                                              aes->gcm.H);
11432                }
11433                /* Reset count. */
11434                aes->aOver = 0;
11435            }
11436            /* Used up some data. */
11437            aSz -= sz;
11438            a += sz;
11439        }
11440
11441        /* Calculate number of blocks of AAD and the leftover. */
11442        blocks = aSz / WC_AES_BLOCK_SIZE;
11443        partial = aSz % WC_AES_BLOCK_SIZE;
11444        if (blocks > 0) {
11445            /* GHASH full blocks now. */
11446        #ifdef HAVE_INTEL_AVX2
11447            if (IS_INTEL_AVX2(intel_flags)) {
11448                AES_GCM_aad_update_avx2(a, blocks * WC_AES_BLOCK_SIZE,
11449                                        AES_TAG(aes), aes->gcm.H);
11450            }
11451            else
11452        #endif
11453        #ifdef HAVE_INTEL_AVX1
11454            if (IS_INTEL_AVX1(intel_flags)) {
11455                AES_GCM_aad_update_avx1(a, blocks * WC_AES_BLOCK_SIZE,
11456                                        AES_TAG(aes), aes->gcm.H);
11457            }
11458            else
11459        #endif
11460            {
11461                AES_GCM_aad_update_aesni(a, blocks * WC_AES_BLOCK_SIZE,
11462                                         AES_TAG(aes), aes->gcm.H);
11463            }
11464            /* Skip over to end of AAD blocks. */
11465            a += blocks * WC_AES_BLOCK_SIZE;
11466        }
11467        if (partial != 0) {
11468            /* Cache the partial block. */
11469            XMEMCPY(AES_LASTGBLOCK(aes), a, (size_t)partial);
11470            aes->aOver = (byte)partial;
11471        }
11472    }
11473    if (endA && (aes->aOver > 0)) {
11474        /* No more AAD coming and we have a partial block. */
11475        /* Fill the rest of the block with zeros. */
11476        XMEMSET(AES_LASTGBLOCK(aes) + aes->aOver, 0,
11477                (size_t)WC_AES_BLOCK_SIZE - aes->aOver);
11478        /* GHASH last AAD block. */
11479    #ifdef HAVE_INTEL_AVX2
11480        if (IS_INTEL_AVX2(intel_flags)) {
11481            AES_GCM_ghash_block_avx2(AES_LASTGBLOCK(aes), AES_TAG(aes),
11482                                     aes->gcm.H);
11483        }
11484        else
11485    #endif
11486    #ifdef HAVE_INTEL_AVX1
11487        if (IS_INTEL_AVX1(intel_flags)) {
11488            AES_GCM_ghash_block_avx1(AES_LASTGBLOCK(aes), AES_TAG(aes),
11489                                     aes->gcm.H);
11490        }
11491        else
11492    #endif
11493        {
11494            AES_GCM_ghash_block_aesni(AES_LASTGBLOCK(aes), AES_TAG(aes),
11495                                      aes->gcm.H);
11496        }
11497        /* Clear partial count for next time through. */
11498        aes->aOver = 0;
11499    }
11500
11501    return 0;
11502}
11503
11504/* Update the AES GCM for encryption with data and/or authentication data.
11505 *
11506 * Implementation uses AVX2, AVX1 or straight AES-NI optimized assembly code.
11507 *
11508 * @param [in, out] aes  AES object.
11509 * @param [out]     c    Buffer to hold cipher text.
11510 * @param [in]      p    Buffer holding plaintext.
11511 * @param [in]      cSz  Length of cipher text/plaintext in bytes.
11512 * @param [in]      a    Buffer holding authentication data.
11513 * @param [in]      aSz  Length of authentication data in bytes.
11514 */
11515static WARN_UNUSED_RESULT int AesGcmEncryptUpdate_aesni(
11516    Aes* aes, byte* c, const byte* p, word32 cSz, const byte* a, word32 aSz)
11517{
11518    word32 blocks;
11519    int partial;
11520    int ret;
11521
11522    ASSERT_SAVED_VECTOR_REGISTERS();
11523
11524    /* Hash in A, the Authentication Data */
11525    ret = AesGcmAadUpdate_aesni(aes, a, aSz, (cSz > 0) && (c != NULL));
11526    if (ret != 0)
11527        return ret;
11528
11529    /* Encrypt plaintext and Hash in C, the Cipher text */
11530    if (cSz != 0 && c != NULL) {
11531        /* Update count of cipher text we have hashed. */
11532        aes->cSz += cSz;
11533        if (aes->cOver > 0) {
11534            /* Calculate amount we can use - fill up the block. */
11535            byte sz = (byte)(WC_AES_BLOCK_SIZE - aes->cOver);
11536            if (sz > cSz) {
11537                sz = (byte)cSz;
11538            }
11539            /* Encrypt some of the plaintext. */
11540            xorbuf(AES_LASTGBLOCK(aes) + aes->cOver, p, sz);
11541            XMEMCPY(c, AES_LASTGBLOCK(aes) + aes->cOver, sz);
11542            /* Update count of unused encrypted counter. */
11543            aes->cOver = (byte)(aes->cOver + sz);
11544            if (aes->cOver == WC_AES_BLOCK_SIZE) {
11545                /* We have filled up the block and can process. */
11546            #ifdef HAVE_INTEL_AVX2
11547                if (IS_INTEL_AVX2(intel_flags)) {
11548                    AES_GCM_ghash_block_avx2(AES_LASTGBLOCK(aes), AES_TAG(aes),
11549                                             aes->gcm.H);
11550                }
11551                else
11552            #endif
11553            #ifdef HAVE_INTEL_AVX1
11554                if (IS_INTEL_AVX1(intel_flags)) {
11555                    AES_GCM_ghash_block_avx1(AES_LASTGBLOCK(aes), AES_TAG(aes),
11556                                             aes->gcm.H);
11557                }
11558                else
11559            #endif
11560                {
11561                    AES_GCM_ghash_block_aesni(AES_LASTGBLOCK(aes), AES_TAG(aes),
11562                                              aes->gcm.H);
11563                }
11564                /* Reset count. */
11565                aes->cOver = 0;
11566            }
11567            /* Used up some data. */
11568            cSz -= sz;
11569            p += sz;
11570            c += sz;
11571        }
11572
11573        /* Calculate number of blocks of plaintext and the leftover. */
11574        blocks = cSz / WC_AES_BLOCK_SIZE;
11575        partial = cSz % WC_AES_BLOCK_SIZE;
11576        if (blocks > 0) {
11577            /* Encrypt and GHASH full blocks now. */
11578        #ifdef HAVE_INTEL_AVX2
11579            if (IS_INTEL_AVX2(intel_flags)) {
11580                AES_GCM_encrypt_update_avx2((byte*)aes->key, (int)aes->rounds,
11581                    c, p, blocks * WC_AES_BLOCK_SIZE, AES_TAG(aes), aes->gcm.H,
11582                    AES_COUNTER(aes));
11583            }
11584            else
11585        #endif
11586        #ifdef HAVE_INTEL_AVX1
11587            if (IS_INTEL_AVX1(intel_flags)) {
11588                AES_GCM_encrypt_update_avx1((byte*)aes->key, (int)aes->rounds,
11589                    c, p, blocks * WC_AES_BLOCK_SIZE, AES_TAG(aes), aes->gcm.H,
11590                    AES_COUNTER(aes));
11591            }
11592            else
11593        #endif
11594            {
11595                AES_GCM_encrypt_update_aesni((byte*)aes->key, (int)aes->rounds,
11596                    c, p, blocks * WC_AES_BLOCK_SIZE, AES_TAG(aes), aes->gcm.H,
11597                    AES_COUNTER(aes));
11598            }
11599            /* Skip over to end of blocks. */
11600            p += blocks * WC_AES_BLOCK_SIZE;
11601            c += blocks * WC_AES_BLOCK_SIZE;
11602        }
11603        if (partial != 0) {
11604            /* Encrypt the counter - XOR in zeros as proxy for plaintext. */
11605            XMEMSET(AES_LASTGBLOCK(aes), 0, WC_AES_BLOCK_SIZE);
11606        #ifdef HAVE_INTEL_AVX2
11607            if (IS_INTEL_AVX2(intel_flags)) {
11608                AES_GCM_encrypt_block_avx2((byte*)aes->key, (int)aes->rounds,
11609                    AES_LASTGBLOCK(aes), AES_LASTGBLOCK(aes), AES_COUNTER(aes));
11610            }
11611            else
11612        #endif
11613        #ifdef HAVE_INTEL_AVX1
11614            if (IS_INTEL_AVX1(intel_flags)) {
11615                AES_GCM_encrypt_block_avx1((byte*)aes->key, (int)aes->rounds,
11616                    AES_LASTGBLOCK(aes), AES_LASTGBLOCK(aes), AES_COUNTER(aes));
11617            }
11618            else
11619        #endif
11620            {
11621                AES_GCM_encrypt_block_aesni((byte*)aes->key, (int)aes->rounds,
11622                    AES_LASTGBLOCK(aes), AES_LASTGBLOCK(aes), AES_COUNTER(aes));
11623            }
11624            /* XOR the remaining plaintext to calculate cipher text.
11625             * Keep cipher text for GHASH of last partial block.
11626             */
11627            xorbuf(AES_LASTGBLOCK(aes), p, (word32)partial);
11628            XMEMCPY(c, AES_LASTGBLOCK(aes), (size_t)partial);
11629            /* Update count of the block used. */
11630            aes->cOver = (byte)partial;
11631        }
11632    }
11633    return 0;
11634}
11635
11636/* Finalize the AES GCM for encryption and calculate the authentication tag.
11637 *
11638 * Calls AVX2, AVX1 or straight AES-NI optimized assembly code.
11639 *
11640 * @param [in, out] aes        AES object.
11641 * @param [in]      authTag    Buffer to hold authentication tag.
11642 * @param [in]      authTagSz  Length of authentication tag in bytes.
11643 * @return  0 on success.
11644 */
11645static WARN_UNUSED_RESULT int AesGcmEncryptFinal_aesni(
11646    Aes* aes, byte* authTag, word32 authTagSz)
11647{
11648    /* AAD block incomplete when > 0 */
11649    byte over = aes->aOver;
11650
11651    ASSERT_SAVED_VECTOR_REGISTERS();
11652
11653    if (aes->cOver > 0) {
11654        /* Cipher text block incomplete. */
11655        over = aes->cOver;
11656    }
11657    if (over > 0) {
11658        /* Fill the rest of the block with zeros. */
11659        XMEMSET(AES_LASTGBLOCK(aes) + over, 0, (size_t)WC_AES_BLOCK_SIZE - over);
11660        /* GHASH last cipher block. */
11661    #ifdef HAVE_INTEL_AVX2
11662        if (IS_INTEL_AVX2(intel_flags)) {
11663            AES_GCM_ghash_block_avx2(AES_LASTGBLOCK(aes), AES_TAG(aes),
11664                                     aes->gcm.H);
11665        }
11666        else
11667    #endif
11668    #ifdef HAVE_INTEL_AVX1
11669        if (IS_INTEL_AVX1(intel_flags)) {
11670            AES_GCM_ghash_block_avx1(AES_LASTGBLOCK(aes), AES_TAG(aes),
11671                                     aes->gcm.H);
11672        }
11673        else
11674    #endif
11675        {
11676            AES_GCM_ghash_block_aesni(AES_LASTGBLOCK(aes), AES_TAG(aes),
11677                                      aes->gcm.H);
11678        }
11679    }
11680    /* Calculate the authentication tag. */
11681#ifdef HAVE_INTEL_AVX2
11682    if (IS_INTEL_AVX2(intel_flags)) {
11683        AES_GCM_encrypt_final_avx2(AES_TAG(aes), authTag, authTagSz, aes->cSz,
11684            aes->aSz, aes->gcm.H, AES_INITCTR(aes));
11685    }
11686    else
11687#endif
11688#ifdef HAVE_INTEL_AVX1
11689    if (IS_INTEL_AVX1(intel_flags)) {
11690        AES_GCM_encrypt_final_avx1(AES_TAG(aes), authTag, authTagSz, aes->cSz,
11691            aes->aSz, aes->gcm.H, AES_INITCTR(aes));
11692    }
11693    else
11694#endif
11695    {
11696        AES_GCM_encrypt_final_aesni(AES_TAG(aes), authTag, authTagSz, aes->cSz,
11697            aes->aSz, aes->gcm.H, AES_INITCTR(aes));
11698    }
11699
11700    return 0;
11701}
11702
11703#if defined(HAVE_AES_DECRYPT) || defined(HAVE_AESGCM_DECRYPT)
11704
11705#ifdef __cplusplus
11706    extern "C" {
11707#endif
11708
11709/* Assembly code implementations in: aes_gcm_asm.S and aes_gcm_x86_asm.S */
11710#ifdef HAVE_INTEL_AVX2
11711extern void AES_GCM_decrypt_update_avx2(const unsigned char* key, int nr,
11712    unsigned char* out, const unsigned char* in, unsigned int nbytes,
11713    unsigned char* tag, unsigned char* h, unsigned char* counter);
11714extern void AES_GCM_decrypt_final_avx2(unsigned char* tag,
11715    const unsigned char* authTag, unsigned int tbytes, unsigned int nbytes,
11716    unsigned int abytes, unsigned char* h, unsigned char* initCtr, int* res);
11717#endif
11718#ifdef HAVE_INTEL_AVX1
11719extern void AES_GCM_decrypt_update_avx1(const unsigned char* key, int nr,
11720    unsigned char* out, const unsigned char* in, unsigned int nbytes,
11721    unsigned char* tag, unsigned char* h, unsigned char* counter);
11722extern void AES_GCM_decrypt_final_avx1(unsigned char* tag,
11723    const unsigned char* authTag, unsigned int tbytes, unsigned int nbytes,
11724    unsigned int abytes, unsigned char* h, unsigned char* initCtr, int* res);
11725#endif
11726extern void AES_GCM_decrypt_update_aesni(const unsigned char* key, int nr,
11727    unsigned char* out, const unsigned char* in, unsigned int nbytes,
11728    unsigned char* tag, unsigned char* h, unsigned char* counter);
11729extern void AES_GCM_decrypt_final_aesni(unsigned char* tag,
11730    const unsigned char* authTag, unsigned int tbytes, unsigned int nbytes,
11731    unsigned int abytes, unsigned char* h, unsigned char* initCtr, int* res);
11732
11733#ifdef __cplusplus
11734    } /* extern "C" */
11735#endif
11736
11737/* Update the AES GCM for decryption with data and/or authentication data.
11738 *
11739 * @param [in, out] aes  AES object.
11740 * @param [out]     p    Buffer to hold plaintext.
11741 * @param [in]      c    Buffer holding cipher text.
11742 * @param [in]      cSz  Length of cipher text/plaintext in bytes.
11743 * @param [in]      a    Buffer holding authentication data.
11744 * @param [in]      aSz  Length of authentication data in bytes.
11745 */
11746static WARN_UNUSED_RESULT int AesGcmDecryptUpdate_aesni(
11747    Aes* aes, byte* p, const byte* c, word32 cSz, const byte* a, word32 aSz)
11748{
11749    word32 blocks;
11750    int partial;
11751    int ret;
11752
11753    ASSERT_SAVED_VECTOR_REGISTERS();
11754
11755    /* Hash in A, the Authentication Data */
11756    ret = AesGcmAadUpdate_aesni(aes, a, aSz, cSz > 0);
11757    if (ret != 0)
11758        return ret;
11759
11760    /* Hash in C, the Cipher text, and decrypt. */
11761    if (cSz != 0 && p != NULL) {
11762        /* Update count of cipher text we have hashed. */
11763        aes->cSz += cSz;
11764        if (aes->cOver > 0) {
11765            /* Calculate amount we can use - fill up the block. */
11766            byte sz = (byte)(WC_AES_BLOCK_SIZE - aes->cOver);
11767            if (sz > cSz) {
11768                sz = (byte)cSz;
11769            }
11770            /* Keep a copy of the cipher text for GHASH. */
11771            XMEMCPY(AES_LASTBLOCK(aes) + aes->cOver, c, sz);
11772            /* Decrypt some of the cipher text. */
11773            xorbuf(AES_LASTGBLOCK(aes) + aes->cOver, c, sz);
11774            XMEMCPY(p, AES_LASTGBLOCK(aes) + aes->cOver, sz);
11775            /* Update count of unused encrypted counter. */
11776            aes->cOver = (byte)(aes->cOver + sz);
11777            if (aes->cOver == WC_AES_BLOCK_SIZE) {
11778                /* We have filled up the block and can process. */
11779            #ifdef HAVE_INTEL_AVX2
11780                if (IS_INTEL_AVX2(intel_flags)) {
11781                    AES_GCM_ghash_block_avx2(AES_LASTBLOCK(aes), AES_TAG(aes),
11782                                             aes->gcm.H);
11783                }
11784                else
11785            #endif
11786            #ifdef HAVE_INTEL_AVX1
11787                if (IS_INTEL_AVX1(intel_flags)) {
11788                    AES_GCM_ghash_block_avx1(AES_LASTBLOCK(aes), AES_TAG(aes),
11789                                             aes->gcm.H);
11790                }
11791                else
11792            #endif
11793                {
11794                    AES_GCM_ghash_block_aesni(AES_LASTBLOCK(aes), AES_TAG(aes),
11795                                              aes->gcm.H);
11796                }
11797                /* Reset count. */
11798                aes->cOver = 0;
11799            }
11800            /* Used up some data. */
11801            cSz -= sz;
11802            c += sz;
11803            p += sz;
11804        }
11805
11806        /* Calculate number of blocks of plaintext and the leftover. */
11807        blocks = cSz / WC_AES_BLOCK_SIZE;
11808        partial = cSz % WC_AES_BLOCK_SIZE;
11809        if (blocks > 0) {
11810            /* Decrypt and GHASH full blocks now. */
11811        #ifdef HAVE_INTEL_AVX2
11812            if (IS_INTEL_AVX2(intel_flags)) {
11813                AES_GCM_decrypt_update_avx2((byte*)aes->key, (int)aes->rounds,
11814                    p, c, blocks * WC_AES_BLOCK_SIZE, AES_TAG(aes), aes->gcm.H,
11815                    AES_COUNTER(aes));
11816            }
11817            else
11818        #endif
11819        #ifdef HAVE_INTEL_AVX1
11820            if (IS_INTEL_AVX1(intel_flags)) {
11821                AES_GCM_decrypt_update_avx1((byte*)aes->key, (int)aes->rounds,
11822                    p, c, blocks * WC_AES_BLOCK_SIZE, AES_TAG(aes), aes->gcm.H,
11823                    AES_COUNTER(aes));
11824            }
11825            else
11826        #endif
11827            {
11828                AES_GCM_decrypt_update_aesni((byte*)aes->key, (int)aes->rounds,
11829                    p, c, blocks * WC_AES_BLOCK_SIZE, AES_TAG(aes), aes->gcm.H,
11830                    AES_COUNTER(aes));
11831            }
11832            /* Skip over to end of blocks. */
11833            c += blocks * WC_AES_BLOCK_SIZE;
11834            p += blocks * WC_AES_BLOCK_SIZE;
11835        }
11836        if (partial != 0) {
11837            /* Encrypt the counter - XOR in zeros as proxy for cipher text. */
11838            XMEMSET(AES_LASTGBLOCK(aes), 0, WC_AES_BLOCK_SIZE);
11839        #ifdef HAVE_INTEL_AVX2
11840            if (IS_INTEL_AVX2(intel_flags)) {
11841                AES_GCM_encrypt_block_avx2((byte*)aes->key, (int)aes->rounds,
11842                    AES_LASTGBLOCK(aes), AES_LASTGBLOCK(aes), AES_COUNTER(aes));
11843            }
11844            else
11845        #endif
11846        #ifdef HAVE_INTEL_AVX1
11847            if (IS_INTEL_AVX1(intel_flags)) {
11848                AES_GCM_encrypt_block_avx1((byte*)aes->key, (int)aes->rounds,
11849                    AES_LASTGBLOCK(aes), AES_LASTGBLOCK(aes), AES_COUNTER(aes));
11850            }
11851            else
11852        #endif
11853            {
11854                AES_GCM_encrypt_block_aesni((byte*)aes->key, (int)aes->rounds,
11855                    AES_LASTGBLOCK(aes), AES_LASTGBLOCK(aes), AES_COUNTER(aes));
11856            }
11857            /* Keep cipher text for GHASH of last partial block. */
11858            XMEMCPY(AES_LASTBLOCK(aes), c, (size_t)partial);
11859            /* XOR the remaining cipher text to calculate plaintext. */
11860            xorbuf(AES_LASTGBLOCK(aes), c, (word32)partial);
11861            XMEMCPY(p, AES_LASTGBLOCK(aes), (size_t)partial);
11862            /* Update count of the block used. */
11863            aes->cOver = (byte)partial;
11864        }
11865    }
11866
11867    return 0;
11868}
11869
11870/* Finalize the AES GCM for decryption and check the authentication tag.
11871 *
11872 * Calls AVX2, AVX1 or straight AES-NI optimized assembly code.
11873 *
11874 * @param [in, out] aes        AES object.
11875 * @param [in]      authTag    Buffer holding authentication tag.
11876 * @param [in]      authTagSz  Length of authentication tag in bytes.
11877 * @return  0 on success.
11878 * @return  AES_GCM_AUTH_E when authentication tag doesn't match calculated
11879 *          value.
11880 */
11881static WARN_UNUSED_RESULT int AesGcmDecryptFinal_aesni(
11882    Aes* aes, const byte* authTag, word32 authTagSz)
11883{
11884    int ret = 0;
11885    int res;
11886    /* AAD block incomplete when > 0 */
11887    byte over = aes->aOver;
11888    byte *lastBlock = AES_LASTGBLOCK(aes);
11889
11890    ASSERT_SAVED_VECTOR_REGISTERS();
11891
11892    if (aes->cOver > 0) {
11893        /* Cipher text block incomplete. */
11894        over = aes->cOver;
11895        lastBlock = AES_LASTBLOCK(aes);
11896    }
11897    if (over > 0) {
11898        /* Zeroize the unused part of the block. */
11899        XMEMSET(lastBlock + over, 0, (size_t)WC_AES_BLOCK_SIZE - over);
11900        /* Hash the last block of cipher text. */
11901    #ifdef HAVE_INTEL_AVX2
11902        if (IS_INTEL_AVX2(intel_flags)) {
11903            AES_GCM_ghash_block_avx2(lastBlock, AES_TAG(aes), aes->gcm.H);
11904        }
11905        else
11906    #endif
11907    #ifdef HAVE_INTEL_AVX1
11908        if (IS_INTEL_AVX1(intel_flags)) {
11909            AES_GCM_ghash_block_avx1(lastBlock, AES_TAG(aes), aes->gcm.H);
11910        }
11911        else
11912    #endif
11913        {
11914            AES_GCM_ghash_block_aesni(lastBlock, AES_TAG(aes), aes->gcm.H);
11915        }
11916    }
11917    /* Calculate and compare the authentication tag. */
11918#ifdef HAVE_INTEL_AVX2
11919    if (IS_INTEL_AVX2(intel_flags)) {
11920        AES_GCM_decrypt_final_avx2(AES_TAG(aes), authTag, authTagSz, aes->cSz,
11921            aes->aSz, aes->gcm.H, AES_INITCTR(aes), &res);
11922    }
11923    else
11924#endif
11925#ifdef HAVE_INTEL_AVX1
11926    if (IS_INTEL_AVX1(intel_flags)) {
11927        AES_GCM_decrypt_final_avx1(AES_TAG(aes), authTag, authTagSz, aes->cSz,
11928            aes->aSz, aes->gcm.H, AES_INITCTR(aes), &res);
11929    }
11930    else
11931#endif
11932    {
11933        AES_GCM_decrypt_final_aesni(AES_TAG(aes), authTag, authTagSz, aes->cSz,
11934            aes->aSz, aes->gcm.H, AES_INITCTR(aes), &res);
11935    }
11936
11937    /* Return error code when calculated doesn't match input. */
11938    if (res == 0) {
11939        ret = AES_GCM_AUTH_E;
11940    }
11941    return ret;
11942}
11943#endif /* HAVE_AES_DECRYPT || HAVE_AESGCM_DECRYPT */
11944#endif /* WOLFSSL_AESNI */
11945
11946#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
11947    !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
11948/* Initialize the AES GCM cipher with an IV. Aarch64 HW Crypto implementations.
11949 *
11950 * @param [in, out] aes   AES object.
11951 * @param [in]      iv    IV/nonce buffer.
11952 * @param [in]      ivSz  Length of IV/nonce data.
11953 */
11954static WARN_UNUSED_RESULT int AesGcmInit_AARCH64(Aes* aes, const byte* iv,
11955    word32 ivSz)
11956{
11957    /* Reset state fields. */
11958    aes->over = 0;
11959    aes->aSz = 0;
11960    aes->cSz = 0;
11961    /* Set tag to all zeros as initial value. */
11962    XMEMSET(AES_TAG(aes), 0, WC_AES_BLOCK_SIZE);
11963    /* Reset counts of AAD and cipher text. */
11964    aes->aOver = 0;
11965    aes->cOver = 0;
11966
11967#ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
11968    if (aes->use_sha3_hw_crypto) {
11969        AES_GCM_init_AARCH64_EOR3((byte*)aes->key, (int)aes->rounds, iv, ivSz,
11970            aes->gcm.H, AES_COUNTER(aes), AES_INITCTR(aes));
11971    }
11972    else
11973#endif
11974    {
11975        AES_GCM_init_AARCH64((byte*)aes->key, (int)aes->rounds, iv, ivSz,
11976            aes->gcm.H, AES_COUNTER(aes), AES_INITCTR(aes));
11977    }
11978
11979    return 0;
11980}
11981
11982/* Update the AES GCM for encryption with authentication data.
11983 *
11984 * Implementation uses AARCH64 optimized assembly code.
11985 *
11986 * @param [in, out] aes   AES object.
11987 * @param [in]      a     Buffer holding authentication data.
11988 * @param [in]      aSz   Length of authentication data in bytes.
11989 * @param [in]      endA  Whether no more authentication data is expected.
11990 */
11991static WARN_UNUSED_RESULT int AesGcmAadUpdate_AARCH64(
11992    Aes* aes, const byte* a, word32 aSz, int endA)
11993{
11994    word32 blocks;
11995    int partial;
11996
11997    if (aSz != 0 && a != NULL) {
11998        /* Total count of AAD updated. */
11999        aes->aSz += aSz;
12000        /* Check if we have unprocessed data. */
12001        if (aes->aOver > 0) {
12002            /* Calculate amount we can use - fill up the block. */
12003            byte sz = (byte)(WC_AES_BLOCK_SIZE - aes->aOver);
12004            if (sz > aSz) {
12005                sz = (byte)aSz;
12006            }
12007            /* Copy extra into last GHASH block array and update count. */
12008            XMEMCPY(AES_LASTGBLOCK(aes) + aes->aOver, a, sz);
12009            aes->aOver = (byte)(aes->aOver + sz);
12010            if (aes->aOver == WC_AES_BLOCK_SIZE) {
12011                /* We have filled up the block and can process. */
12012            #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
12013                if (aes->use_sha3_hw_crypto) {
12014                    AES_GCM_ghash_block_AARCH64_EOR3(AES_LASTGBLOCK(aes),
12015                        AES_TAG(aes), aes->gcm.H);
12016                }
12017                else
12018            #endif
12019                {
12020                    AES_GCM_ghash_block_AARCH64(AES_LASTGBLOCK(aes),
12021                        AES_TAG(aes), aes->gcm.H);
12022                }
12023                /* Reset count. */
12024                aes->aOver = 0;
12025            }
12026            /* Used up some data. */
12027            aSz -= sz;
12028            a += sz;
12029        }
12030
12031        /* Calculate number of blocks of AAD and the leftover. */
12032        blocks = aSz / WC_AES_BLOCK_SIZE;
12033        partial = aSz % WC_AES_BLOCK_SIZE;
12034        if (blocks > 0) {
12035            /* GHASH full blocks now. */
12036        #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
12037            if (aes->use_sha3_hw_crypto) {
12038                AES_GCM_aad_update_AARCH64_EOR3(a, blocks * WC_AES_BLOCK_SIZE,
12039                    AES_TAG(aes), aes->gcm.H);
12040            }
12041            else
12042        #endif
12043            {
12044                AES_GCM_aad_update_AARCH64(a, blocks * WC_AES_BLOCK_SIZE,
12045                    AES_TAG(aes), aes->gcm.H);
12046            }
12047            /* Skip over to end of AAD blocks. */
12048            a += blocks * WC_AES_BLOCK_SIZE;
12049        }
12050        if (partial != 0) {
12051            /* Cache the partial block. */
12052            XMEMCPY(AES_LASTGBLOCK(aes), a, (size_t)partial);
12053            aes->aOver = (byte)partial;
12054        }
12055    }
12056    if (endA && (aes->aOver > 0)) {
12057        /* No more AAD coming and we have a partial block. */
12058        /* Fill the rest of the block with zeros. */
12059        XMEMSET(AES_LASTGBLOCK(aes) + aes->aOver, 0,
12060                (size_t)WC_AES_BLOCK_SIZE - aes->aOver);
12061        /* GHASH last AAD block. */
12062    #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
12063        if (aes->use_sha3_hw_crypto) {
12064            AES_GCM_ghash_block_AARCH64_EOR3(AES_LASTGBLOCK(aes),
12065                AES_TAG(aes), aes->gcm.H);
12066        }
12067        else
12068    #endif
12069        {
12070            AES_GCM_ghash_block_AARCH64(AES_LASTGBLOCK(aes),
12071                AES_TAG(aes), aes->gcm.H);
12072        }
12073        /* Clear partial count for next time through. */
12074        aes->aOver = 0;
12075    }
12076
12077    return 0;
12078}
12079
12080/* Update the AES GCM for encryption with data and/or authentication data.
12081 *
12082 * Implementation uses AARCH64 optimized assembly code.
12083 *
12084 * @param [in, out] aes  AES object.
12085 * @param [out]     c    Buffer to hold cipher text.
12086 * @param [in]      p    Buffer holding plaintext.
12087 * @param [in]      cSz  Length of cipher text/plaintext in bytes.
12088 * @param [in]      a    Buffer holding authentication data.
12089 * @param [in]      aSz  Length of authentication data in bytes.
12090 */
12091static WARN_UNUSED_RESULT int AesGcmEncryptUpdate_AARCH64(
12092    Aes* aes, byte* c, const byte* p, word32 cSz, const byte* a, word32 aSz)
12093{
12094    word32 blocks;
12095    int partial;
12096    int ret;
12097
12098    /* Hash in A, the Authentication Data */
12099    ret = AesGcmAadUpdate_AARCH64(aes, a, aSz, (cSz > 0) && (c != NULL));
12100    if (ret != 0)
12101        return ret;
12102
12103    /* Encrypt plaintext and Hash in C, the Cipher text */
12104    if (cSz != 0 && c != NULL) {
12105        /* Update count of cipher text we have hashed. */
12106        aes->cSz += cSz;
12107        if (aes->cOver > 0) {
12108            /* Calculate amount we can use - fill up the block. */
12109            byte sz = (byte)(WC_AES_BLOCK_SIZE - aes->cOver);
12110            if (sz > cSz) {
12111                sz = (byte)cSz;
12112            }
12113            /* Encrypt some of the plaintext. */
12114            xorbuf(AES_LASTGBLOCK(aes) + aes->cOver, p, sz);
12115            XMEMCPY(c, AES_LASTGBLOCK(aes) + aes->cOver, sz);
12116            /* Update count of unused encrypted counter. */
12117            aes->cOver = (byte)(aes->cOver + sz);
12118            if (aes->cOver == WC_AES_BLOCK_SIZE) {
12119                /* We have filled up the block and can process. */
12120            #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
12121                if (aes->use_sha3_hw_crypto) {
12122                    AES_GCM_ghash_block_AARCH64_EOR3(AES_LASTGBLOCK(aes),
12123                        AES_TAG(aes), aes->gcm.H);
12124                }
12125                else
12126            #endif
12127                {
12128                    AES_GCM_ghash_block_AARCH64(AES_LASTGBLOCK(aes),
12129                        AES_TAG(aes), aes->gcm.H);
12130                }
12131                /* Reset count. */
12132                aes->cOver = 0;
12133            }
12134            /* Used up some data. */
12135            cSz -= sz;
12136            p += sz;
12137            c += sz;
12138        }
12139
12140        /* Calculate number of blocks of plaintext and the leftover. */
12141        blocks = cSz / WC_AES_BLOCK_SIZE;
12142        partial = cSz % WC_AES_BLOCK_SIZE;
12143        if (blocks > 0) {
12144            /* Encrypt and GHASH full blocks now. */
12145        #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
12146            if (aes->use_sha3_hw_crypto) {
12147                AES_GCM_encrypt_update_AARCH64_EOR3((byte*)aes->key,
12148                    (int)aes->rounds, c, p, blocks * WC_AES_BLOCK_SIZE,
12149                    AES_TAG(aes), aes->gcm.H, AES_COUNTER(aes));
12150            }
12151            else
12152        #endif
12153            {
12154                AES_GCM_encrypt_update_AARCH64((byte*)aes->key,
12155                    (int)aes->rounds, c, p, blocks * WC_AES_BLOCK_SIZE,
12156                    AES_TAG(aes), aes->gcm.H, AES_COUNTER(aes));
12157            }
12158            /* Skip over to end of blocks. */
12159            p += blocks * WC_AES_BLOCK_SIZE;
12160            c += blocks * WC_AES_BLOCK_SIZE;
12161        }
12162        if (partial != 0) {
12163            /* Encrypt the counter - XOR in zeros as proxy for plaintext. */
12164            XMEMSET(AES_LASTGBLOCK(aes), 0, WC_AES_BLOCK_SIZE);
12165        #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
12166            if (aes->use_sha3_hw_crypto) {
12167                AES_GCM_encrypt_block_AARCH64_EOR3((byte*)aes->key,
12168                    (int)aes->rounds, AES_LASTGBLOCK(aes), AES_LASTGBLOCK(aes),
12169                    AES_COUNTER(aes));
12170            }
12171            else
12172        #endif
12173            {
12174                AES_GCM_encrypt_block_AARCH64((byte*)aes->key, (int)aes->rounds,
12175                    AES_LASTGBLOCK(aes), AES_LASTGBLOCK(aes), AES_COUNTER(aes));
12176            }
12177            /* XOR the remaining plaintext to calculate cipher text.
12178             * Keep cipher text for GHASH of last partial block.
12179             */
12180            xorbuf(AES_LASTGBLOCK(aes), p, (word32)partial);
12181            XMEMCPY(c, AES_LASTGBLOCK(aes), (size_t)partial);
12182            /* Update count of the block used. */
12183            aes->cOver = (byte)partial;
12184        }
12185    }
12186    return 0;
12187}
12188
12189/* Finalize the AES GCM for encryption and calculate the authentication tag.
12190 *
12191 * Calls ARCH64 optimized assembly code.
12192 *
12193 * @param [in, out] aes        AES object.
12194 * @param [in]      authTag    Buffer to hold authentication tag.
12195 * @param [in]      authTagSz  Length of authentication tag in bytes.
12196 * @return  0 on success.
12197 */
12198static WARN_UNUSED_RESULT int AesGcmEncryptFinal_AARCH64(Aes* aes,
12199    byte* authTag, word32 authTagSz)
12200{
12201    /* AAD block incomplete when > 0 */
12202    byte over = aes->aOver;
12203
12204    ASSERT_SAVED_VECTOR_REGISTERS();
12205
12206    if (aes->cOver > 0) {
12207        /* Cipher text block incomplete. */
12208        over = aes->cOver;
12209    }
12210    if (over > 0) {
12211        /* Fill the rest of the block with zeros. */
12212        XMEMSET(AES_LASTGBLOCK(aes) + over, 0,
12213            (size_t)WC_AES_BLOCK_SIZE - over);
12214        /* GHASH last cipher block. */
12215    #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
12216        if (aes->use_sha3_hw_crypto) {
12217            AES_GCM_ghash_block_AARCH64_EOR3(AES_LASTGBLOCK(aes), AES_TAG(aes),
12218                aes->gcm.H);
12219        }
12220        else
12221    #endif
12222        {
12223            AES_GCM_ghash_block_AARCH64(AES_LASTGBLOCK(aes), AES_TAG(aes),
12224                aes->gcm.H);
12225        }
12226    }
12227    /* Calculate the authentication tag. */
12228#ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
12229    if (aes->use_sha3_hw_crypto) {
12230        AES_GCM_encrypt_final_AARCH64_EOR3(AES_TAG(aes), authTag, authTagSz,
12231            aes->cSz, aes->aSz, aes->gcm.H, AES_INITCTR(aes));
12232    }
12233    else
12234#endif
12235    {
12236        AES_GCM_encrypt_final_AARCH64(AES_TAG(aes), authTag, authTagSz,
12237            aes->cSz, aes->aSz, aes->gcm.H, AES_INITCTR(aes));
12238    }
12239
12240    return 0;
12241}
12242
12243#if defined(HAVE_AES_DECRYPT) || defined(HAVE_AESGCM_DECRYPT)
12244/* Update the AES GCM for decryption with data and/or authentication data.
12245 *
12246 * @param [in, out] aes  AES object.
12247 * @param [out]     p    Buffer to hold plaintext.
12248 * @param [in]      c    Buffer holding cipher text.
12249 * @param [in]      cSz  Length of cipher text/plaintext in bytes.
12250 * @param [in]      a    Buffer holding authentication data.
12251 * @param [in]      aSz  Length of authentication data in bytes.
12252 */
12253static WARN_UNUSED_RESULT int AesGcmDecryptUpdate_AARCH64(Aes* aes, byte* p,
12254    const byte* c, word32 cSz, const byte* a, word32 aSz)
12255{
12256    word32 blocks;
12257    int partial;
12258    int ret;
12259
12260    /* Hash in A, the Authentication Data */
12261    ret = AesGcmAadUpdate_AARCH64(aes, a, aSz, cSz > 0);
12262    if (ret != 0)
12263        return ret;
12264
12265    /* Hash in C, the Cipher text, and decrypt. */
12266    if (cSz != 0 && p != NULL) {
12267        /* Update count of cipher text we have hashed. */
12268        aes->cSz += cSz;
12269        if (aes->cOver > 0) {
12270            /* Calculate amount we can use - fill up the block. */
12271            byte sz = (byte)(WC_AES_BLOCK_SIZE - aes->cOver);
12272            if (sz > cSz) {
12273                sz = (byte)cSz;
12274            }
12275            /* Keep a copy of the cipher text for GHASH. */
12276            XMEMCPY(AES_LASTBLOCK(aes) + aes->cOver, c, sz);
12277            /* Decrypt some of the cipher text. */
12278            xorbuf(AES_LASTGBLOCK(aes) + aes->cOver, c, sz);
12279            XMEMCPY(p, AES_LASTGBLOCK(aes) + aes->cOver, sz);
12280            /* Update count of unused encrypted counter. */
12281            aes->cOver = (byte)(aes->cOver + sz);
12282            if (aes->cOver == WC_AES_BLOCK_SIZE) {
12283                /* We have filled up the block and can process. */
12284            #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
12285                if (aes->use_sha3_hw_crypto) {
12286                    AES_GCM_ghash_block_AARCH64_EOR3(AES_LASTBLOCK(aes),
12287                        AES_TAG(aes), aes->gcm.H);
12288                }
12289                else
12290            #endif
12291                {
12292                    AES_GCM_ghash_block_AARCH64(AES_LASTBLOCK(aes),
12293                        AES_TAG(aes), aes->gcm.H);
12294                }
12295                /* Reset count. */
12296                aes->cOver = 0;
12297            }
12298            /* Used up some data. */
12299            cSz -= sz;
12300            c += sz;
12301            p += sz;
12302        }
12303
12304        /* Calculate number of blocks of plaintext and the leftover. */
12305        blocks = cSz / WC_AES_BLOCK_SIZE;
12306        partial = cSz % WC_AES_BLOCK_SIZE;
12307        if (blocks > 0) {
12308            /* Decrypt and GHASH full blocks now. */
12309        #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
12310            if (aes->use_sha3_hw_crypto) {
12311                AES_GCM_decrypt_update_AARCH64_EOR3((byte*)aes->key,
12312                    (int)aes->rounds, p, c, blocks * WC_AES_BLOCK_SIZE,
12313                    AES_TAG(aes), aes->gcm.H, AES_COUNTER(aes));
12314            }
12315            else
12316        #endif
12317            {
12318                AES_GCM_decrypt_update_AARCH64((byte*)aes->key,
12319                    (int)aes->rounds, p, c, blocks * WC_AES_BLOCK_SIZE,
12320                    AES_TAG(aes), aes->gcm.H, AES_COUNTER(aes));
12321            }
12322            /* Skip over to end of blocks. */
12323            c += blocks * WC_AES_BLOCK_SIZE;
12324            p += blocks * WC_AES_BLOCK_SIZE;
12325        }
12326        if (partial != 0) {
12327            /* Encrypt the counter - XOR in zeros as proxy for cipher text. */
12328            XMEMSET(AES_LASTGBLOCK(aes), 0, WC_AES_BLOCK_SIZE);
12329        #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
12330            if (aes->use_sha3_hw_crypto) {
12331                AES_GCM_encrypt_block_AARCH64_EOR3((byte*)aes->key,
12332                    (int)aes->rounds, AES_LASTGBLOCK(aes), AES_LASTGBLOCK(aes),
12333                    AES_COUNTER(aes));
12334            }
12335            else
12336        #endif
12337            {
12338                AES_GCM_encrypt_block_AARCH64((byte*)aes->key, (int)aes->rounds,
12339                    AES_LASTGBLOCK(aes), AES_LASTGBLOCK(aes), AES_COUNTER(aes));
12340            }
12341            /* Keep cipher text for GHASH of last partial block. */
12342            XMEMCPY(AES_LASTBLOCK(aes), c, (size_t)partial);
12343            /* XOR the remaining cipher text to calculate plaintext. */
12344            xorbuf(AES_LASTGBLOCK(aes), c, (word32)partial);
12345            XMEMCPY(p, AES_LASTGBLOCK(aes), (size_t)partial);
12346            /* Update count of the block used. */
12347            aes->cOver = (byte)partial;
12348        }
12349    }
12350
12351    return 0;
12352}
12353
12354/* Finalize the AES GCM for decryption and check the authentication tag.
12355 *
12356 * Calls AVX2, AVX1 or straight AES-NI optimized assembly code.
12357 *
12358 * @param [in, out] aes        AES object.
12359 * @param [in]      authTag    Buffer holding authentication tag.
12360 * @param [in]      authTagSz  Length of authentication tag in bytes.
12361 * @return  0 on success.
12362 * @return  AES_GCM_AUTH_E when authentication tag doesn't match calculated
12363 *          value.
12364 */
12365static WARN_UNUSED_RESULT int AesGcmDecryptFinal_AARCH64(
12366    Aes* aes, const byte* authTag, word32 authTagSz)
12367{
12368    int ret = 0;
12369    int res;
12370    /* AAD block incomplete when > 0 */
12371    byte over = aes->aOver;
12372    byte *lastBlock = AES_LASTGBLOCK(aes);
12373
12374    ASSERT_SAVED_VECTOR_REGISTERS();
12375
12376    if (aes->cOver > 0) {
12377        /* Cipher text block incomplete. */
12378        over = aes->cOver;
12379        lastBlock = AES_LASTBLOCK(aes);
12380    }
12381    if (over > 0) {
12382        /* Zeroize the unused part of the block. */
12383        XMEMSET(lastBlock + over, 0, (size_t)WC_AES_BLOCK_SIZE - over);
12384        /* Hash the last block of cipher text. */
12385    #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
12386        if (aes->use_sha3_hw_crypto) {
12387            AES_GCM_ghash_block_AARCH64_EOR3(lastBlock, AES_TAG(aes),
12388                aes->gcm.H);
12389        }
12390        else
12391    #endif
12392        {
12393            AES_GCM_ghash_block_AARCH64(lastBlock, AES_TAG(aes), aes->gcm.H);
12394        }
12395    }
12396    /* Calculate and compare the authentication tag. */
12397#ifdef WOLFSSL_ARMASM_CRYPTO_SHA3
12398    if (aes->use_sha3_hw_crypto) {
12399        AES_GCM_decrypt_final_AARCH64_EOR3(AES_TAG(aes), authTag, authTagSz,
12400            aes->cSz, aes->aSz, aes->gcm.H, AES_INITCTR(aes), &res);
12401    }
12402    else
12403#endif
12404    {
12405        AES_GCM_decrypt_final_AARCH64(AES_TAG(aes), authTag, authTagSz,
12406            aes->cSz, aes->aSz, aes->gcm.H, AES_INITCTR(aes), &res);
12407    }
12408
12409    /* Return error code when calculated doesn't match input. */
12410    if (res == 0) {
12411        ret = AES_GCM_AUTH_E;
12412    }
12413    return ret;
12414}
12415#endif
12416#endif
12417
12418/* Initialize an AES GCM cipher for encryption or decryption.
12419 *
12420 * Must call wc_AesInit() before calling this function.
12421 * Call wc_AesGcmSetIV() before calling this function to generate part of IV.
12422 * Call wc_AesGcmSetExtIV() before calling this function to cache IV.
12423 *
12424 * @param [in, out] aes   AES object.
12425 * @param [in]      key   Buffer holding key.
12426 * @param [in]      len   Length of key in bytes.
12427 * @param [in]      iv    Buffer holding IV/nonce.
12428 * @param [in]      ivSz  Length of IV/nonce in bytes.
12429 * @return  0 on success.
12430 * @return  BAD_FUNC_ARG when aes is NULL, or a length is non-zero but buffer
12431 *          is NULL, or the IV is NULL and no previous IV has been set.
12432 * @return  MEMORY_E when dynamic memory allocation fails. (WOLFSSL_SMALL_STACK)
12433 */
12434int wc_AesGcmInit(Aes* aes, const byte* key, word32 len, const byte* iv,
12435    word32 ivSz)
12436{
12437    int ret = 0;
12438
12439    /* Check validity of parameters. */
12440    if ((aes == NULL) || ((len > 0) && (key == NULL)) ||
12441            ((ivSz == 0) && (iv != NULL)) ||
12442            ((ivSz > 0) && (iv == NULL))) {
12443        ret = BAD_FUNC_ARG;
12444    }
12445
12446#if defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI)
12447    if ((ret == 0) && (aes->streamData == NULL)) {
12448        /* Allocate buffers for streaming. */
12449        aes->streamData_sz = 5 * WC_AES_BLOCK_SIZE;
12450        aes->streamData = (byte*)XMALLOC(aes->streamData_sz, aes->heap,
12451                                                              DYNAMIC_TYPE_AES);
12452        if (aes->streamData == NULL) {
12453            ret = MEMORY_E;
12454        }
12455    }
12456#endif
12457
12458    /* Set the key if passed in. */
12459    if ((ret == 0) && (key != NULL)) {
12460        ret = wc_AesGcmSetKey(aes, key, len);
12461    }
12462
12463    if (ret == 0) {
12464        /* Set the IV passed in if it is smaller than a block. */
12465        if ((iv != NULL) && (ivSz <= WC_AES_BLOCK_SIZE)) {
12466            XMEMMOVE((byte*)aes->reg, iv, ivSz);
12467            aes->nonceSz = ivSz;
12468        }
12469        /* No IV passed in, check for cached IV. */
12470        if ((iv == NULL) && (aes->nonceSz != 0)) {
12471            /* Use the cached copy. */
12472            iv = (byte*)aes->reg;
12473            ivSz = aes->nonceSz;
12474        }
12475
12476        if (iv != NULL) {
12477            /* Initialize with the IV. */
12478
12479        #ifdef WOLFSSL_AESNI
12480            if (aes->use_aesni) {
12481                SAVE_VECTOR_REGISTERS(return _svr_ret;);
12482                ret = AesGcmInit_aesni(aes, iv, ivSz);
12483                RESTORE_VECTOR_REGISTERS();
12484            }
12485            else
12486        #elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
12487              !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
12488            if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) {
12489                ret = AesGcmInit_AARCH64(aes, iv, ivSz);
12490            }
12491            else
12492        #endif /* WOLFSSL_AESNI */
12493            {
12494                ret = AesGcmInit_C(aes, iv, ivSz);
12495            }
12496
12497            if (ret == 0)
12498                aes->nonceSet = 1;
12499        }
12500    }
12501
12502    return ret;
12503}
12504
12505/* Initialize an AES GCM cipher for encryption.
12506 *
12507 * Must call wc_AesInit() before calling this function.
12508 *
12509 * @param [in, out] aes   AES object.
12510 * @param [in]      key   Buffer holding key.
12511 * @param [in]      len   Length of key in bytes.
12512 * @param [in]      iv    Buffer holding IV/nonce.
12513 * @param [in]      ivSz  Length of IV/nonce in bytes.
12514 * @return  0 on success.
12515 * @return  BAD_FUNC_ARG when aes is NULL, or a length is non-zero but buffer
12516 *          is NULL, or the IV is NULL and no previous IV has been set.
12517 */
12518int wc_AesGcmEncryptInit(Aes* aes, const byte* key, word32 len, const byte* iv,
12519    word32 ivSz)
12520{
12521    return wc_AesGcmInit(aes, key, len, iv, ivSz);
12522}
12523
12524/* Initialize an AES GCM cipher for encryption. Get IV.
12525 *
12526 * Must call wc_AesGcmSetIV() to generate part of IV before calling this
12527 * function.
12528 * Must call wc_AesInit() before calling this function.
12529 *
12530 * See wc_AesGcmEncrypt_ex() for non-streaming version of getting IV out.
12531 *
12532 * @param [in, out] aes   AES object.
12533 * @param [in]      key   Buffer holding key.
12534 * @param [in]      len   Length of key in bytes.
12535 * @param [in]      iv    Buffer holding IV/nonce.
12536 * @param [in]      ivSz  Length of IV/nonce in bytes.
12537 * @return  0 on success.
12538 * @return  BAD_FUNC_ARG when aes is NULL, key length is non-zero but key
12539 *          is NULL, or the IV is NULL or ivOutSz is not the same as cached
12540 *          nonce size.
12541 */
12542int wc_AesGcmEncryptInit_ex(Aes* aes, const byte* key, word32 len, byte* ivOut,
12543    word32 ivOutSz)
12544{
12545    int ret;
12546
12547    /* Check validity of parameters. */
12548    if ((aes == NULL) || (ivOut == NULL) || (ivOutSz != aes->nonceSz)) {
12549        ret = BAD_FUNC_ARG;
12550    }
12551    else {
12552        /* Copy out the IV including generated part for decryption. */
12553        XMEMCPY(ivOut, aes->reg, ivOutSz);
12554        /* Initialize AES GCM cipher with key and cached Iv. */
12555        ret = wc_AesGcmInit(aes, key, len, NULL, 0);
12556    }
12557
12558    return ret;
12559}
12560
12561/* Update the AES GCM for encryption with data and/or authentication data. */
12562int wc_AesGcmEncryptUpdate(Aes* aes, byte* out, const byte* in, word32 sz,
12563    const byte* authIn, word32 authInSz)
12564{
12565    int ret = 0;
12566
12567    /* Check validity of parameters. */
12568    if ((aes == NULL) || ((authInSz > 0) && (authIn == NULL)) || ((sz > 0) &&
12569            ((out == NULL) || (in == NULL)))) {
12570        ret = BAD_FUNC_ARG;
12571    }
12572
12573    /* Check key has been set. */
12574    if ((ret == 0) && (!aes->gcmKeySet)) {
12575        ret = MISSING_KEY;
12576    }
12577    /* Check IV has been set. */
12578    if ((ret == 0) && (!aes->nonceSet)) {
12579        ret = MISSING_IV;
12580    }
12581
12582    if ((ret == 0) && aes->ctrSet && (aes->aSz == 0) && (aes->cSz == 0)) {
12583        aes->invokeCtr[0]++;
12584        if (aes->invokeCtr[0] == 0) {
12585            aes->invokeCtr[1]++;
12586            if (aes->invokeCtr[1] == 0)
12587                ret = AES_GCM_OVERFLOW_E;
12588        }
12589    }
12590
12591    if (ret == 0) {
12592        /* Encrypt with AAD and/or plaintext. */
12593
12594    #ifdef WOLFSSL_AESNI
12595        if (aes->use_aesni) {
12596            SAVE_VECTOR_REGISTERS(return _svr_ret;);
12597            ret = AesGcmEncryptUpdate_aesni(aes, out, in, sz, authIn, authInSz);
12598            RESTORE_VECTOR_REGISTERS();
12599        }
12600        else
12601    #elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
12602          !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
12603        if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) {
12604            ret = AesGcmEncryptUpdate_AARCH64(aes, out, in, sz, authIn,
12605                authInSz);
12606        }
12607        else
12608    #endif
12609        {
12610            /* Encrypt the plaintext. */
12611            ret = AesGcmCryptUpdate_C(aes, out, in, sz);
12612            if (ret == 0) {
12613                /* Update the authentication tag with any authentication data and the
12614                 * new cipher text. */
12615                GHASH_UPDATE(aes, authIn, authInSz, out, sz);
12616            }
12617        }
12618    }
12619
12620    return ret;
12621}
12622
12623/* Finalize the AES GCM for encryption and return the authentication tag.
12624 *
12625 * Must set key and IV before calling this function.
12626 * Must call wc_AesGcmInit() before calling this function.
12627 *
12628 * @param [in, out] aes        AES object.
12629 * @param [out]     authTag    Buffer to hold authentication tag.
12630 * @param [in]      authTagSz  Length of authentication tag in bytes.
12631 * @return  0 on success.
12632 */
12633int wc_AesGcmEncryptFinal(Aes* aes, byte* authTag, word32 authTagSz)
12634{
12635    int ret = 0;
12636
12637    /* Check validity of parameters. */
12638    if ((aes == NULL) || (authTag == NULL) || (authTagSz > WC_AES_BLOCK_SIZE) ||
12639            (authTagSz < WOLFSSL_MIN_AUTH_TAG_SZ)) {
12640        ret = BAD_FUNC_ARG;
12641    }
12642
12643    /* Check key has been set. */
12644    if ((ret == 0) && (!aes->gcmKeySet)) {
12645        ret = MISSING_KEY;
12646    }
12647    /* Check IV has been set. */
12648    if ((ret == 0) && (!aes->nonceSet)) {
12649        ret = MISSING_IV;
12650    }
12651
12652    if (ret == 0) {
12653        /* Calculate authentication tag. */
12654    #ifdef WOLFSSL_AESNI
12655        if (aes->use_aesni) {
12656            SAVE_VECTOR_REGISTERS(return _svr_ret;);
12657            ret = AesGcmEncryptFinal_aesni(aes, authTag, authTagSz);
12658            RESTORE_VECTOR_REGISTERS();
12659        }
12660        else
12661    #elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
12662          !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
12663        if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) {
12664            ret = AesGcmEncryptFinal_AARCH64(aes, authTag, authTagSz);
12665        }
12666        else
12667    #endif
12668        {
12669            ret = AesGcmFinal_C(aes, authTag, authTagSz);
12670        }
12671    }
12672
12673    if ((ret == 0) && aes->ctrSet) {
12674        IncCtr((byte*)aes->reg, aes->nonceSz);
12675    }
12676
12677    return ret;
12678}
12679
12680#if defined(HAVE_AES_DECRYPT) || defined(HAVE_AESGCM_DECRYPT)
12681/* Initialize an AES GCM cipher for decryption.
12682 *
12683 * Must call wc_AesInit() before calling this function.
12684 *
12685 * Call wc_AesGcmSetExtIV() before calling this function to use FIPS external IV
12686 * instead.
12687 *
12688 * @param [in, out] aes   AES object.
12689 * @param [in]      key   Buffer holding key.
12690 * @param [in]      len   Length of key in bytes.
12691 * @param [in]      iv    Buffer holding IV/nonce.
12692 * @param [in]      ivSz  Length of IV/nonce in bytes.
12693 * @return  0 on success.
12694 * @return  BAD_FUNC_ARG when aes is NULL, or a length is non-zero but buffer
12695 *          is NULL, or the IV is NULL and no previous IV has been set.
12696 */
12697int wc_AesGcmDecryptInit(Aes* aes, const byte* key, word32 len, const byte* iv,
12698    word32 ivSz)
12699{
12700    return wc_AesGcmInit(aes, key, len, iv, ivSz);
12701}
12702
12703/* Update the AES GCM for decryption with data and/or authentication data. */
12704int wc_AesGcmDecryptUpdate(Aes* aes, byte* out, const byte* in, word32 sz,
12705    const byte* authIn, word32 authInSz)
12706{
12707    int ret = 0;
12708
12709    /* Check validity of parameters. */
12710    if ((aes == NULL) || ((authInSz > 0) && (authIn == NULL)) || ((sz > 0) &&
12711            ((out == NULL) || (in == NULL)))) {
12712        ret = BAD_FUNC_ARG;
12713    }
12714
12715    /* Check key has been set. */
12716    if ((ret == 0) && (!aes->gcmKeySet)) {
12717        ret = MISSING_KEY;
12718    }
12719    /* Check IV has been set. */
12720    if ((ret == 0) && (!aes->nonceSet)) {
12721        ret = MISSING_IV;
12722    }
12723
12724    if (ret == 0) {
12725        /* Decrypt with AAD and/or cipher text. */
12726    #ifdef WOLFSSL_AESNI
12727        if (aes->use_aesni) {
12728            SAVE_VECTOR_REGISTERS(return _svr_ret;);
12729            ret = AesGcmDecryptUpdate_aesni(aes, out, in, sz, authIn, authInSz);
12730            RESTORE_VECTOR_REGISTERS();
12731        }
12732        else
12733    #elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
12734          !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
12735        if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) {
12736            ret = AesGcmDecryptUpdate_AARCH64(aes, out, in, sz, authIn,
12737                authInSz);
12738        }
12739        else
12740    #endif
12741        {
12742            /* Update the authentication tag with any authentication data and
12743             * cipher text. */
12744            GHASH_UPDATE(aes, authIn, authInSz, in, sz);
12745            /* Decrypt the cipher text. */
12746            ret = AesGcmCryptUpdate_C(aes, out, in, sz);
12747        }
12748    }
12749
12750    return ret;
12751}
12752
12753/* Finalize the AES GCM for decryption and check the authentication tag.
12754 *
12755 * Must set key and IV before calling this function.
12756 * Must call wc_AesGcmInit() before calling this function.
12757 *
12758 * @param [in, out] aes        AES object.
12759 * @param [in]      authTag    Buffer holding authentication tag.
12760 * @param [in]      authTagSz  Length of authentication tag in bytes.
12761 * @return  0 on success.
12762 */
12763int wc_AesGcmDecryptFinal(Aes* aes, const byte* authTag, word32 authTagSz)
12764{
12765    int ret = 0;
12766
12767    /* Check validity of parameters. */
12768    if ((aes == NULL) || (authTag == NULL) || (authTagSz > WC_AES_BLOCK_SIZE) ||
12769            (authTagSz < WOLFSSL_MIN_AUTH_TAG_SZ)) {
12770        ret = BAD_FUNC_ARG;
12771    }
12772
12773    /* Check key has been set. */
12774    if ((ret == 0) && (!aes->gcmKeySet)) {
12775        ret = MISSING_KEY;
12776    }
12777    /* Check IV has been set. */
12778    if ((ret == 0) && (!aes->nonceSet)) {
12779        ret = MISSING_IV;
12780    }
12781
12782    if (ret == 0) {
12783        /* Calculate authentication tag and compare with one passed in.. */
12784    #ifdef WOLFSSL_AESNI
12785        if (aes->use_aesni) {
12786            SAVE_VECTOR_REGISTERS(return _svr_ret;);
12787            ret = AesGcmDecryptFinal_aesni(aes, authTag, authTagSz);
12788            RESTORE_VECTOR_REGISTERS();
12789        }
12790        else
12791    #elif defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
12792          !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
12793        if (aes->use_aes_hw_crypto && aes->use_pmull_hw_crypto) {
12794            ret = AesGcmDecryptFinal_AARCH64(aes, authTag, authTagSz);
12795        }
12796        else
12797    #endif
12798        {
12799            ALIGN32 byte calcTag[WC_AES_BLOCK_SIZE];
12800            /* Calculate authentication tag. */
12801            ret = AesGcmFinal_C(aes, calcTag, WC_AES_BLOCK_SIZE);
12802            if (ret == 0) {
12803                /* Check calculated tag matches the one passed in. */
12804                if (ConstantCompare(authTag, calcTag, (int)authTagSz) != 0) {
12805                    ret = AES_GCM_AUTH_E;
12806                }
12807            }
12808        }
12809    }
12810
12811    return ret;
12812}
12813#endif /* HAVE_AES_DECRYPT || HAVE_AESGCM_DECRYPT */
12814#endif /* WOLFSSL_AESGCM_STREAM */
12815#endif /* WOLFSSL_XILINX_CRYPT */
12816#endif /* end of block for AESGCM implementation selection */
12817
12818
12819/* Common to all, abstract functions that build off of lower level AESGCM
12820 * functions */
12821#ifndef WC_NO_RNG
12822
12823static WARN_UNUSED_RESULT WC_INLINE int CheckAesGcmIvSize(int ivSz) {
12824    return (ivSz == GCM_NONCE_MIN_SZ ||
12825            ivSz == GCM_NONCE_MID_SZ ||
12826            ivSz == GCM_NONCE_MAX_SZ);
12827}
12828
12829
12830int wc_AesGcmSetExtIV(Aes* aes, const byte* iv, word32 ivSz)
12831{
12832    int ret = 0;
12833
12834    if (aes == NULL || iv == NULL || !CheckAesGcmIvSize((int)ivSz)) {
12835        ret = BAD_FUNC_ARG;
12836    }
12837
12838    if (ret == 0) {
12839        XMEMCPY((byte*)aes->reg, iv, ivSz);
12840
12841        /* If the IV is 96, allow for a 2^64 invocation counter.
12842         * For any other size for the nonce, limit the invocation
12843         * counter to 32-bits. (SP 800-38D 8.3) */
12844        aes->invokeCtr[0] = 0;
12845        aes->invokeCtr[1] = (ivSz == GCM_NONCE_MID_SZ) ? 0 : 0xFFFFFFFF;
12846    #ifdef WOLFSSL_AESGCM_STREAM
12847        aes->ctrSet = 1;
12848    #endif
12849        aes->nonceSz = ivSz;
12850    }
12851
12852    return ret;
12853}
12854
12855
12856int wc_AesGcmSetIV(Aes* aes, word32 ivSz,
12857                   const byte* ivFixed, word32 ivFixedSz,
12858                   WC_RNG* rng)
12859{
12860    int ret = 0;
12861
12862    if (aes == NULL || rng == NULL || !CheckAesGcmIvSize((int)ivSz) ||
12863        (ivFixed == NULL && ivFixedSz != 0) ||
12864        (ivFixed != NULL && ivFixedSz != AES_IV_FIXED_SZ)) {
12865
12866        ret = BAD_FUNC_ARG;
12867    }
12868
12869    if (ret == 0) {
12870        byte* iv = (byte*)aes->reg;
12871
12872        if (ivFixedSz)
12873            XMEMCPY(iv, ivFixed, ivFixedSz);
12874
12875        ret = wc_RNG_GenerateBlock(rng, iv + ivFixedSz, ivSz - ivFixedSz);
12876    }
12877
12878    if (ret == 0) {
12879        /* If the IV is 96, allow for a 2^64 invocation counter.
12880         * For any other size for the nonce, limit the invocation
12881         * counter to 32-bits. (SP 800-38D 8.3) */
12882        aes->invokeCtr[0] = 0;
12883        aes->invokeCtr[1] = (ivSz == GCM_NONCE_MID_SZ) ? 0 : 0xFFFFFFFF;
12884    #ifdef WOLFSSL_AESGCM_STREAM
12885        aes->ctrSet = 1;
12886    #endif
12887        aes->nonceSz = ivSz;
12888    }
12889
12890    return ret;
12891}
12892
12893
12894int wc_AesGcmEncrypt_ex(Aes* aes, byte* out, const byte* in, word32 sz,
12895                        byte* ivOut, word32 ivOutSz,
12896                        byte* authTag, word32 authTagSz,
12897                        const byte* authIn, word32 authInSz)
12898{
12899    int ret = 0;
12900
12901    if (aes == NULL || (sz != 0 && (in == NULL || out == NULL)) ||
12902        ivOut == NULL || ivOutSz != aes->nonceSz ||
12903        (authIn == NULL && authInSz != 0)) {
12904
12905        ret = BAD_FUNC_ARG;
12906    }
12907
12908    if (ret == 0) {
12909        aes->invokeCtr[0]++;
12910        if (aes->invokeCtr[0] == 0) {
12911            aes->invokeCtr[1]++;
12912            if (aes->invokeCtr[1] == 0)
12913                ret = AES_GCM_OVERFLOW_E;
12914        }
12915    }
12916
12917    if (ret == 0) {
12918        XMEMCPY(ivOut, aes->reg, ivOutSz);
12919        ret = wc_AesGcmEncrypt(aes, out, in, sz,
12920                               (byte*)aes->reg, ivOutSz,
12921                               authTag, authTagSz,
12922                               authIn, authInSz);
12923        if (ret == 0)
12924            IncCtr((byte*)aes->reg, ivOutSz);
12925    }
12926
12927    return ret;
12928}
12929
12930int wc_Gmac(const byte* key, word32 keySz, byte* iv, word32 ivSz,
12931            const byte* authIn, word32 authInSz,
12932            byte* authTag, word32 authTagSz, WC_RNG* rng)
12933{
12934    WC_DECLARE_VAR(aes, Aes, 1, 0);
12935    int ret;
12936
12937    if (key == NULL || iv == NULL || (authIn == NULL && authInSz != 0) ||
12938        authTag == NULL || authTagSz == 0 || rng == NULL) {
12939
12940        return BAD_FUNC_ARG;
12941    }
12942
12943#ifdef WOLFSSL_SMALL_STACK
12944    aes = wc_AesNew(NULL, INVALID_DEVID, &ret);
12945#else
12946    ret = wc_AesInit(aes, NULL, INVALID_DEVID);
12947#endif
12948    if (ret != 0)
12949        return ret;
12950
12951    ret = wc_AesGcmSetKey(aes, key, keySz);
12952    if (ret == 0)
12953        ret = wc_AesGcmSetIV(aes, ivSz, NULL, 0, rng);
12954    if (ret == 0)
12955        ret = wc_AesGcmEncrypt_ex(aes, NULL, NULL, 0, iv, ivSz,
12956                                  authTag, authTagSz, authIn, authInSz);
12957
12958#ifdef WOLFSSL_SMALL_STACK
12959    wc_AesDelete(aes, NULL);
12960#else
12961    wc_AesFree(aes);
12962#endif
12963
12964    return ret;
12965}
12966
12967int wc_GmacVerify(const byte* key, word32 keySz,
12968                  const byte* iv, word32 ivSz,
12969                  const byte* authIn, word32 authInSz,
12970                  const byte* authTag, word32 authTagSz)
12971{
12972    int ret;
12973#ifdef HAVE_AES_DECRYPT
12974    WC_DECLARE_VAR(aes, Aes, 1, 0);
12975
12976    if (key == NULL || iv == NULL || (authIn == NULL && authInSz != 0) ||
12977        authTag == NULL || authTagSz == 0 || authTagSz > WC_AES_BLOCK_SIZE) {
12978
12979        return BAD_FUNC_ARG;
12980    }
12981
12982#ifdef WOLFSSL_SMALL_STACK
12983    aes = wc_AesNew(NULL, INVALID_DEVID, &ret);
12984#else
12985    ret = wc_AesInit(aes, NULL, INVALID_DEVID);
12986#endif
12987    if (ret == 0) {
12988        ret = wc_AesGcmSetKey(aes, key, keySz);
12989        if (ret == 0)
12990            ret = wc_AesGcmDecrypt(aes, NULL, NULL, 0, iv, ivSz,
12991                                  authTag, authTagSz, authIn, authInSz);
12992
12993    }
12994#ifdef WOLFSSL_SMALL_STACK
12995    wc_AesDelete(aes, NULL);
12996#else
12997    wc_AesFree(aes);
12998#endif
12999#else
13000    (void)key;
13001    (void)keySz;
13002    (void)iv;
13003    (void)ivSz;
13004    (void)authIn;
13005    (void)authInSz;
13006    (void)authTag;
13007    (void)authTagSz;
13008    ret = NOT_COMPILED_IN;
13009#endif
13010    return ret;
13011}
13012
13013#endif /* WC_NO_RNG */
13014
13015
13016int wc_GmacSetKey(Gmac* gmac, const byte* key, word32 len)
13017{
13018    if (gmac == NULL || key == NULL) {
13019        return BAD_FUNC_ARG;
13020    }
13021    return wc_AesGcmSetKey(&gmac->aes, key, len);
13022}
13023
13024
13025int wc_GmacUpdate(Gmac* gmac, const byte* iv, word32 ivSz,
13026                              const byte* authIn, word32 authInSz,
13027                              byte* authTag, word32 authTagSz)
13028{
13029    if (gmac == NULL) {
13030        return BAD_FUNC_ARG;
13031    }
13032
13033    return wc_AesGcmEncrypt(&gmac->aes, NULL, NULL, 0, iv, ivSz,
13034                                         authTag, authTagSz, authIn, authInSz);
13035}
13036
13037#endif /* HAVE_AESGCM */
13038
13039#ifdef HAVE_AESCCM
13040
13041int wc_AesCcmSetKey(Aes* aes, const byte* key, word32 keySz)
13042{
13043    if (!((keySz == 16) || (keySz == 24) || (keySz == 32)))
13044        return BAD_FUNC_ARG;
13045
13046    return wc_AesSetKey(aes, key, keySz, NULL, AES_ENCRYPTION);
13047}
13048
13049
13050/* Checks if the tag size is an accepted value based on RFC 3610 section 2
13051 * returns 0 if tag size is ok
13052 */
13053int wc_AesCcmCheckTagSize(int sz)
13054{
13055    /* values here are from RFC 3610 section 2 */
13056    if (sz != 4 && sz != 6 && sz != 8 && sz != 10 && sz != 12 && sz != 14
13057            && sz != 16) {
13058        WOLFSSL_MSG("Bad auth tag size AES-CCM");
13059        return BAD_FUNC_ARG;
13060    }
13061    return 0;
13062}
13063
13064#if defined(WOLFSSL_RISCV_ASM)
13065    /* implementation located in wolfcrypt/src/port/riscv/riscv-64-aes.c */
13066
13067#elif defined(HAVE_COLDFIRE_SEC)
13068    #error "Coldfire SEC doesn't currently support AES-CCM mode"
13069
13070#elif defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES) && \
13071        !defined(WOLFSSL_QNX_CAAM)
13072    /* implemented in wolfcrypt/src/port/caam_aes.c */
13073
13074#elif defined(WOLFSSL_SILABS_SE_ACCEL)
13075    /* implemented in wolfcrypt/src/port/silabs/silabs_aes.c */
13076int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
13077                   const byte* nonce, word32 nonceSz,
13078                   byte* authTag, word32 authTagSz,
13079                   const byte* authIn, word32 authInSz)
13080{
13081    return wc_AesCcmEncrypt_silabs(
13082        aes, out, in, inSz,
13083        nonce, nonceSz,
13084        authTag, authTagSz,
13085        authIn, authInSz);
13086}
13087
13088#ifdef HAVE_AES_DECRYPT
13089int  wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
13090                   const byte* nonce, word32 nonceSz,
13091                   const byte* authTag, word32 authTagSz,
13092                   const byte* authIn, word32 authInSz)
13093{
13094    return wc_AesCcmDecrypt_silabs(
13095        aes, out, in, inSz,
13096        nonce, nonceSz,
13097        authTag, authTagSz,
13098        authIn, authInSz);
13099}
13100#endif
13101#elif defined(FREESCALE_LTC)
13102
13103/* return 0 on success */
13104int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
13105                   const byte* nonce, word32 nonceSz,
13106                   byte* authTag, word32 authTagSz,
13107                   const byte* authIn, word32 authInSz)
13108{
13109    byte *key;
13110    word32 keySize;
13111    status_t status;
13112
13113    /* sanity check on arguments */
13114    /* note, LTC_AES_EncryptTagCcm() doesn't allow null src or dst
13115     * ptrs even if inSz is zero (ltc_aes_ccm_check_input_args()), so
13116     * don't allow it here either.
13117     */
13118    if (aes == NULL || out == NULL || in == NULL || nonce == NULL
13119            || authTag == NULL || nonceSz < 7 || nonceSz > 13) {
13120        return BAD_FUNC_ARG;
13121    }
13122
13123    if (wc_AesCcmCheckTagSize(authTagSz) != 0) {
13124        return BAD_FUNC_ARG;
13125    }
13126
13127    key = (byte*)aes->key;
13128
13129    status = wc_AesGetKeySize(aes, &keySize);
13130    if (status != 0) {
13131        return status;
13132    }
13133
13134    status = wolfSSL_CryptHwMutexLock();
13135    if (status != 0)
13136        return status;
13137
13138    status = LTC_AES_EncryptTagCcm(LTC_BASE, in, out, inSz,
13139        nonce, nonceSz, authIn, authInSz, key, keySize, authTag, authTagSz);
13140    wolfSSL_CryptHwMutexUnLock();
13141
13142    return (kStatus_Success == status) ? 0 : BAD_FUNC_ARG;
13143}
13144
13145#ifdef HAVE_AES_DECRYPT
13146int  wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
13147                   const byte* nonce, word32 nonceSz,
13148                   const byte* authTag, word32 authTagSz,
13149                   const byte* authIn, word32 authInSz)
13150{
13151    byte *key;
13152    word32 keySize;
13153    status_t status;
13154
13155    /* sanity check on arguments */
13156    if (aes == NULL || out == NULL || in == NULL || nonce == NULL
13157            || authTag == NULL || nonceSz < 7 || nonceSz > 13) {
13158        return BAD_FUNC_ARG;
13159    }
13160
13161    key = (byte*)aes->key;
13162
13163    status = wc_AesGetKeySize(aes, &keySize);
13164    if (status != 0) {
13165        return status;
13166    }
13167
13168    status = wolfSSL_CryptHwMutexLock();
13169    if (status != 0)
13170        return status;
13171    status = LTC_AES_DecryptTagCcm(LTC_BASE, in, out, inSz,
13172        nonce, nonceSz, authIn, authInSz, key, keySize, authTag, authTagSz);
13173    wolfSSL_CryptHwMutexUnLock();
13174
13175    if (status != kStatus_Success) {
13176        XMEMSET(out, 0, inSz);
13177        return AES_CCM_AUTH_E;
13178    }
13179    return 0;
13180}
13181#endif /* HAVE_AES_DECRYPT */
13182
13183#else
13184
13185/* Software CCM */
13186static WARN_UNUSED_RESULT int roll_x(
13187    Aes* aes, const byte* in, word32 inSz, byte* out)
13188{
13189    int ret;
13190
13191    /* process the bulk of the data */
13192    while (inSz >= WC_AES_BLOCK_SIZE) {
13193        xorbuf(out, in, WC_AES_BLOCK_SIZE);
13194        in += WC_AES_BLOCK_SIZE;
13195        inSz -= WC_AES_BLOCK_SIZE;
13196
13197        /* wc_AesCcmEncrypt(), wc_AesCcmDecrypt(), and roll_auth() only call
13198         * roll_x() after the AES cache lines are already hot -- no need to
13199         * absorb additional prefetch overhead here.
13200         */
13201        ret = AesEncrypt_preFetchOpt(aes, out, out, &never_prefetch);
13202        if (ret != 0)
13203            return ret;
13204    }
13205
13206    /* process remainder of the data */
13207    if (inSz > 0) {
13208        xorbuf(out, in, inSz);
13209        /* wc_AesCcmEncrypt(), wc_AesCcmDecrypt(), and roll_auth() only call
13210         * roll_x() after the AES cache lines are already hot -- no need to
13211         * absorb additional prefetch overhead here.
13212         */
13213        ret = AesEncrypt_preFetchOpt(aes, out, out, &never_prefetch);
13214        if (ret != 0)
13215            return ret;
13216    }
13217
13218    return 0;
13219}
13220
13221static WARN_UNUSED_RESULT int roll_auth(
13222    Aes* aes, const byte* in, word32 inSz, byte* out)
13223{
13224    word32 authLenSz;
13225    word32 remainder;
13226    int ret;
13227
13228    /* encode the length in */
13229    if (inSz <= 0xFEFF) {
13230        authLenSz = 2;
13231        out[0] ^= (byte)(inSz >> 8);
13232        out[1] ^= (byte)inSz;
13233    }
13234    else {
13235        authLenSz = 6;
13236        out[0] ^= 0xFF;
13237        out[1] ^= 0xFE;
13238        out[2] ^= (byte)(inSz >> 24);
13239        out[3] ^= (byte)(inSz >> 16);
13240        out[4] ^= (byte)(inSz >>  8);
13241        out[5] ^= (byte)inSz;
13242    }
13243    /* Note, the protocol handles auth data up to 2^64, but we are
13244     * using 32-bit sizes right now, so the bigger data isn't handled
13245     * else {}
13246     */
13247
13248    /* start fill out the rest of the first block */
13249    remainder = WC_AES_BLOCK_SIZE - authLenSz;
13250    if (inSz >= remainder) {
13251        /* plenty of bulk data to fill the remainder of this block */
13252        xorbuf(out + authLenSz, in, remainder);
13253        inSz -= remainder;
13254        in += remainder;
13255    }
13256    else {
13257        /* not enough bulk data, copy what is available, and pad zero */
13258        xorbuf(out + authLenSz, in, inSz);
13259        inSz = 0;
13260    }
13261    /* wc_AesCcmEncrypt() and wc_AesCcmDecrypt() only call roll_auth() after the
13262     * AES cache lines are already hot -- no need to absorb additional prefetch
13263     * overhead here.
13264     */
13265    ret = AesEncrypt_preFetchOpt(aes, out, out, &never_prefetch);
13266
13267    if ((ret == 0) && (inSz > 0)) {
13268        ret = roll_x(aes, in, inSz, out);
13269    }
13270
13271    return ret;
13272}
13273
13274
13275static WC_INLINE void AesCcmCtrInc(byte* B, word32 lenSz)
13276{
13277    word32 i;
13278
13279    for (i = 0; i < lenSz; i++) {
13280        if (++B[WC_AES_BLOCK_SIZE - 1 - i] != 0) return;
13281    }
13282}
13283
13284#ifdef WOLFSSL_AESNI
13285static WC_INLINE void AesCcmCtrIncSet4(byte* B, word32 lenSz)
13286{
13287    word32 i;
13288
13289    /* B+1 = B */
13290    XMEMCPY(B + WC_AES_BLOCK_SIZE * 1, B, WC_AES_BLOCK_SIZE);
13291    /* B+2,B+3 = B,B+1 */
13292    XMEMCPY(B + WC_AES_BLOCK_SIZE * 2, B, WC_AES_BLOCK_SIZE * 2);
13293
13294    for (i = 0; i < lenSz; i++) {
13295        if (++B[WC_AES_BLOCK_SIZE * 2 - 1 - i] != 0) break;
13296    }
13297    B[WC_AES_BLOCK_SIZE * 3 - 1] = (byte)(B[WC_AES_BLOCK_SIZE * 3 - 1] + 2U);
13298    if (B[WC_AES_BLOCK_SIZE * 3 - 1] < 2U) {
13299        for (i = 1; i < lenSz; i++) {
13300            if (++B[WC_AES_BLOCK_SIZE * 3 - 1 - i] != 0) break;
13301        }
13302    }
13303    B[WC_AES_BLOCK_SIZE * 4 - 1] = (byte)(B[WC_AES_BLOCK_SIZE * 4 - 1] + 3U);
13304    if (B[WC_AES_BLOCK_SIZE * 4 - 1] < 3U) {
13305        for (i = 1; i < lenSz; i++) {
13306            if (++B[WC_AES_BLOCK_SIZE * 4 - 1 - i] != 0) break;
13307        }
13308    }
13309}
13310
13311static WC_INLINE void AesCcmCtrInc4(byte* B, word32 lenSz)
13312{
13313    word32 i;
13314
13315    B[WC_AES_BLOCK_SIZE - 1] = (byte)(B[WC_AES_BLOCK_SIZE - 1] + 4U);
13316    if (B[WC_AES_BLOCK_SIZE - 1] < 4U) {
13317        for (i = 1; i < lenSz; i++) {
13318            if (++B[WC_AES_BLOCK_SIZE - 1 - i] != 0) break;
13319        }
13320    }
13321}
13322#endif
13323
13324/* Software AES - CCM Encrypt */
13325/* return 0 on success */
13326int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
13327                   const byte* nonce, word32 nonceSz,
13328                   byte* authTag, word32 authTagSz,
13329                   const byte* authIn, word32 authInSz)
13330{
13331#ifdef WOLFSSL_AESNI
13332    ALIGN128 byte A[WC_AES_BLOCK_SIZE * 4];
13333    ALIGN128 byte B[WC_AES_BLOCK_SIZE * 4];
13334#else
13335    byte A[WC_AES_BLOCK_SIZE];
13336    byte B[WC_AES_BLOCK_SIZE];
13337#endif
13338    byte lenSz;
13339    word32 i;
13340    byte mask = 0xFF;
13341    const word32 wordSz = (word32)sizeof(word32);
13342    int ret;
13343
13344    /* sanity check on arguments */
13345    if (aes == NULL || (inSz != 0 && (in == NULL || out == NULL)) ||
13346        nonce == NULL || authTag == NULL || nonceSz < 7 || nonceSz > 13 ||
13347            authTagSz > WC_AES_BLOCK_SIZE)
13348        return BAD_FUNC_ARG;
13349
13350    /* Sanity check on authIn to prevent segfault in xorbuf() where
13351     * variable 'in' is dereferenced as the mask 'm' in misc.c */
13352    if (authIn == NULL && authInSz > 0)
13353        return BAD_FUNC_ARG;
13354
13355    /* sanity check on tag size */
13356    if (wc_AesCcmCheckTagSize((int)authTagSz) != 0) {
13357        return BAD_FUNC_ARG;
13358    }
13359
13360#ifdef WOLF_CRYPTO_CB
13361    #ifndef WOLF_CRYPTO_CB_FIND
13362    if (aes->devId != INVALID_DEVID)
13363    #endif
13364    {
13365        int crypto_cb_ret =
13366            wc_CryptoCb_AesCcmEncrypt(aes, out, in, inSz, nonce, nonceSz,
13367                                      authTag, authTagSz, authIn, authInSz);
13368        if (crypto_cb_ret != WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE))
13369            return crypto_cb_ret;
13370        /* fall-through when unavailable */
13371    }
13372#endif
13373
13374    XMEMSET(A, 0, sizeof(A));
13375    XMEMCPY(B+1, nonce, nonceSz);
13376    lenSz = (byte)(WC_AES_BLOCK_SIZE - 1U - nonceSz);
13377    B[0] = (byte)((authInSz > 0 ? 64 : 0)
13378                  + (8 * (((byte)authTagSz - 2) / 2))
13379                  + (lenSz - 1));
13380    for (i = 0; i < lenSz; i++) {
13381        if (mask && i >= wordSz)
13382            mask = 0x00;
13383        B[WC_AES_BLOCK_SIZE - 1 - i] = (byte)((inSz >> ((8 * i) & mask)) & mask);
13384    }
13385
13386#ifdef WOLFSSL_CHECK_MEM_ZERO
13387    wc_MemZero_Add("wc_AesCcmEncrypt B", B, sizeof(B));
13388#endif
13389
13390    VECTOR_REGISTERS_PUSH;
13391    /* note this wc_AesEncrypt() will perform cache prefetches if needed, so
13392     * that the later encrypt ops don't need to.
13393     */
13394    ret = wc_AesEncrypt(aes, B, A);
13395#ifdef WOLFSSL_CHECK_MEM_ZERO
13396    if (ret == 0)
13397        wc_MemZero_Add("wc_AesCcmEncrypt A", A, sizeof(A));
13398#endif
13399
13400    if ((ret == 0) && (authInSz > 0))
13401        ret = roll_auth(aes, authIn, authInSz, A);
13402
13403    if ((ret == 0) && (inSz > 0))
13404        ret = roll_x(aes, in, inSz, A);
13405
13406    if (ret == 0) {
13407        XMEMCPY(authTag, A, authTagSz);
13408
13409        B[0] = (byte)(lenSz - 1U);
13410        for (i = 0; i < lenSz; i++)
13411            B[WC_AES_BLOCK_SIZE - 1 - i] = 0;
13412        ret = AesEncrypt_preFetchOpt(aes, B, A, &never_prefetch);
13413    }
13414
13415    if (ret == 0) {
13416        xorbuf(authTag, A, authTagSz);
13417        B[15] = 1;
13418    }
13419#ifdef WOLFSSL_AESNI
13420    if ((ret == 0) && aes->use_aesni) {
13421        while (inSz >= WC_AES_BLOCK_SIZE * 4) {
13422            AesCcmCtrIncSet4(B, lenSz);
13423
13424            AES_ECB_encrypt_AESNI(B, A, WC_AES_BLOCK_SIZE * 4, (byte*)aes->key,
13425                            (int)aes->rounds);
13426
13427            xorbuf(A, in, WC_AES_BLOCK_SIZE * 4);
13428            XMEMCPY(out, A, WC_AES_BLOCK_SIZE * 4);
13429
13430            inSz -= WC_AES_BLOCK_SIZE * 4;
13431            in += WC_AES_BLOCK_SIZE * 4;
13432            out += WC_AES_BLOCK_SIZE * 4;
13433
13434            AesCcmCtrInc4(B, lenSz);
13435        }
13436    }
13437#endif
13438    if (ret == 0) {
13439        while (inSz >= WC_AES_BLOCK_SIZE) {
13440            ret = AesEncrypt_preFetchOpt(aes, B, A, &never_prefetch);
13441            if (ret != 0)
13442                break;
13443            xorbuf(A, in, WC_AES_BLOCK_SIZE);
13444            XMEMCPY(out, A, WC_AES_BLOCK_SIZE);
13445
13446            AesCcmCtrInc(B, lenSz);
13447            inSz -= WC_AES_BLOCK_SIZE;
13448            in += WC_AES_BLOCK_SIZE;
13449            out += WC_AES_BLOCK_SIZE;
13450        }
13451    }
13452    if ((ret == 0) && (inSz > 0)) {
13453        ret = AesEncrypt_preFetchOpt(aes, B, A, &never_prefetch);
13454    }
13455    if ((ret == 0) && (inSz > 0)) {
13456        xorbuf(A, in, inSz);
13457        XMEMCPY(out, A, inSz);
13458    }
13459
13460    ForceZero(A, sizeof(A));
13461    ForceZero(B, sizeof(B));
13462
13463#ifdef WOLFSSL_CHECK_MEM_ZERO
13464    wc_MemZero_Check(A, sizeof(A));
13465    wc_MemZero_Check(B, sizeof(B));
13466#endif
13467
13468    VECTOR_REGISTERS_POP;
13469
13470    return ret;
13471}
13472
13473#ifdef HAVE_AES_DECRYPT
13474/* Software AES - CCM Decrypt */
13475int  wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
13476                   const byte* nonce, word32 nonceSz,
13477                   const byte* authTag, word32 authTagSz,
13478                   const byte* authIn, word32 authInSz)
13479{
13480#ifdef WOLFSSL_AESNI
13481    ALIGN128 byte B[WC_AES_BLOCK_SIZE * 4];
13482    ALIGN128 byte A[WC_AES_BLOCK_SIZE * 4];
13483#else
13484    byte A[WC_AES_BLOCK_SIZE];
13485    byte B[WC_AES_BLOCK_SIZE];
13486#endif
13487    byte* o;
13488    byte lenSz;
13489    word32 i, oSz;
13490    byte mask = 0xFF;
13491    const word32 wordSz = (word32)sizeof(word32);
13492    int ret = 0;
13493#ifdef WC_AES_HAVE_PREFETCH_ARG
13494    int did_prefetches = 0;
13495#endif
13496
13497    /* sanity check on arguments */
13498    if (aes == NULL || (inSz != 0 && (in == NULL || out == NULL)) ||
13499        nonce == NULL || authTag == NULL || nonceSz < 7 || nonceSz > 13 ||
13500        authTagSz > WC_AES_BLOCK_SIZE)
13501        return BAD_FUNC_ARG;
13502
13503    /* Sanity check on authIn to prevent segfault in xorbuf() where
13504     * variable 'in' is dereferenced as the mask 'm' in misc.c */
13505    if (authIn == NULL && authInSz > 0)
13506        return BAD_FUNC_ARG;
13507
13508    /* sanity check on tag size */
13509    if (wc_AesCcmCheckTagSize((int)authTagSz) != 0) {
13510        return BAD_FUNC_ARG;
13511    }
13512
13513#ifdef WOLF_CRYPTO_CB
13514    #ifndef WOLF_CRYPTO_CB_FIND
13515    if (aes->devId != INVALID_DEVID)
13516    #endif
13517    {
13518        int crypto_cb_ret =
13519            wc_CryptoCb_AesCcmDecrypt(aes, out, in, inSz, nonce, nonceSz,
13520            authTag, authTagSz, authIn, authInSz);
13521        if (crypto_cb_ret != WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE))
13522            return crypto_cb_ret;
13523        /* fall-through when unavailable */
13524    }
13525#endif
13526
13527    o = out;
13528    oSz = inSz;
13529    XMEMSET(A, 0, sizeof A);
13530    XMEMCPY(B+1, nonce, nonceSz);
13531    lenSz = (byte)(WC_AES_BLOCK_SIZE - 1U - nonceSz);
13532
13533    B[0] = (byte)(lenSz - 1U);
13534    for (i = 0; i < lenSz; i++)
13535        B[WC_AES_BLOCK_SIZE - 1 - i] = 0;
13536    B[15] = 1;
13537
13538#ifdef WOLFSSL_CHECK_MEM_ZERO
13539    wc_MemZero_Add("wc_AesCcmEncrypt A", A, sizeof(A));
13540    wc_MemZero_Add("wc_AesCcmEncrypt B", B, sizeof(B));
13541#endif
13542
13543    VECTOR_REGISTERS_PUSH;
13544
13545#ifdef WOLFSSL_AESNI
13546    if (aes->use_aesni) {
13547        while (oSz >= WC_AES_BLOCK_SIZE * 4) {
13548            AesCcmCtrIncSet4(B, lenSz);
13549
13550            AES_ECB_encrypt_AESNI(B, A, WC_AES_BLOCK_SIZE * 4, (byte*)aes->key,
13551                            (int)aes->rounds);
13552
13553            xorbuf(A, in, WC_AES_BLOCK_SIZE * 4);
13554            XMEMCPY(o, A, WC_AES_BLOCK_SIZE * 4);
13555
13556            oSz -= WC_AES_BLOCK_SIZE * 4;
13557            in += WC_AES_BLOCK_SIZE * 4;
13558            o += WC_AES_BLOCK_SIZE * 4;
13559
13560            AesCcmCtrInc4(B, lenSz);
13561        }
13562    }
13563#endif
13564
13565    while (oSz >= WC_AES_BLOCK_SIZE) {
13566        ret = AesEncrypt_preFetchOpt(aes, B, A, &did_prefetches);
13567        if (ret != 0)
13568            break;
13569        xorbuf(A, in, WC_AES_BLOCK_SIZE);
13570        XMEMCPY(o, A, WC_AES_BLOCK_SIZE);
13571        AesCcmCtrInc(B, lenSz);
13572        oSz -= WC_AES_BLOCK_SIZE;
13573        in += WC_AES_BLOCK_SIZE;
13574        o += WC_AES_BLOCK_SIZE;
13575    }
13576
13577    if ((ret == 0) && (inSz > 0))
13578        ret = AesEncrypt_preFetchOpt(aes, B, A, &did_prefetches);
13579
13580    if ((ret == 0) && (inSz > 0)) {
13581        xorbuf(A, in, oSz);
13582        XMEMCPY(o, A, oSz);
13583        for (i = 0; i < lenSz; i++)
13584            B[WC_AES_BLOCK_SIZE - 1 - i] = 0;
13585        ret = AesEncrypt_preFetchOpt(aes, B, A, &did_prefetches);
13586    }
13587
13588    if (ret == 0) {
13589        o = out;
13590        oSz = inSz;
13591
13592        B[0] = (byte)((authInSz > 0 ? 64 : 0)
13593                      + (8 * (((byte)authTagSz - 2) / 2))
13594                      + (lenSz - 1));
13595        for (i = 0; i < lenSz; i++) {
13596            if (mask && i >= wordSz)
13597                mask = 0x00;
13598            B[WC_AES_BLOCK_SIZE - 1 - i] = (byte)((inSz >> ((8 * i) & mask)) & mask);
13599        }
13600
13601        ret = AesEncrypt_preFetchOpt(aes, B, A, &did_prefetches);
13602    }
13603
13604    if (ret == 0) {
13605        if (authInSz > 0)
13606            ret = roll_auth(aes, authIn, authInSz, A);
13607    }
13608    if ((ret == 0) && (inSz > 0))
13609        ret = roll_x(aes, o, oSz, A);
13610
13611    if (ret == 0) {
13612        B[0] = (byte)(lenSz - 1U);
13613        for (i = 0; i < lenSz; i++)
13614            B[WC_AES_BLOCK_SIZE - 1 - i] = 0;
13615        ret = AesEncrypt_preFetchOpt(aes, B, B, &did_prefetches);
13616    }
13617
13618    if (ret == 0)
13619        xorbuf(A, B, authTagSz);
13620
13621    if (ret == 0) {
13622        if (ConstantCompare(A, authTag, (int)authTagSz) != 0) {
13623            /* If the authTag check fails, don't keep the decrypted data.
13624             * Unfortunately, you need the decrypted data to calculate the
13625             * check value. */
13626            #if defined(HAVE_FIPS_VERSION) && (HAVE_FIPS_VERSION >= 2) &&   \
13627                        defined(ACVP_VECTOR_TESTING)
13628            WOLFSSL_MSG("Preserve output for vector responses");
13629            #else
13630            if (inSz > 0)
13631                XMEMSET(out, 0, inSz);
13632            #endif
13633            ret = AES_CCM_AUTH_E;
13634        }
13635    }
13636
13637    ForceZero(A, sizeof(A));
13638    ForceZero(B, sizeof(B));
13639    o = NULL;
13640
13641#ifdef WOLFSSL_CHECK_MEM_ZERO
13642    wc_MemZero_Check(A, sizeof(A));
13643    wc_MemZero_Check(B, sizeof(B));
13644#endif
13645
13646    VECTOR_REGISTERS_POP;
13647
13648    return ret;
13649}
13650
13651#endif /* HAVE_AES_DECRYPT */
13652#endif /* software CCM */
13653
13654/* abstract functions that call lower level AESCCM functions */
13655#ifndef WC_NO_RNG
13656
13657int wc_AesCcmSetNonce(Aes* aes, const byte* nonce, word32 nonceSz)
13658{
13659    int ret = 0;
13660
13661    if (aes == NULL || nonce == NULL ||
13662        nonceSz < CCM_NONCE_MIN_SZ || nonceSz > CCM_NONCE_MAX_SZ) {
13663
13664        ret = BAD_FUNC_ARG;
13665    }
13666
13667    if (ret == 0) {
13668        XMEMCPY(aes->reg, nonce, nonceSz);
13669        aes->nonceSz = nonceSz;
13670
13671        /* Invocation counter should be 2^61 */
13672        aes->invokeCtr[0] = 0;
13673        aes->invokeCtr[1] = 0xE0000000;
13674    }
13675
13676    return ret;
13677}
13678
13679
13680int wc_AesCcmEncrypt_ex(Aes* aes, byte* out, const byte* in, word32 sz,
13681                        byte* ivOut, word32 ivOutSz,
13682                        byte* authTag, word32 authTagSz,
13683                        const byte* authIn, word32 authInSz)
13684{
13685    int ret = 0;
13686
13687    if (aes == NULL || out == NULL ||
13688        (in == NULL && sz != 0) ||
13689        ivOut == NULL ||
13690        (authIn == NULL && authInSz != 0) ||
13691        (ivOutSz != aes->nonceSz)) {
13692
13693        ret = BAD_FUNC_ARG;
13694    }
13695
13696    if (ret == 0) {
13697        aes->invokeCtr[0]++;
13698        if (aes->invokeCtr[0] == 0) {
13699            aes->invokeCtr[1]++;
13700            if (aes->invokeCtr[1] == 0)
13701                ret = AES_CCM_OVERFLOW_E;
13702        }
13703    }
13704
13705    if (ret == 0) {
13706        ret = wc_AesCcmEncrypt(aes, out, in, sz,
13707                               (byte*)aes->reg, aes->nonceSz,
13708                               authTag, authTagSz,
13709                               authIn, authInSz);
13710        if (ret == 0) {
13711            XMEMCPY(ivOut, aes->reg, aes->nonceSz);
13712            IncCtr((byte*)aes->reg, aes->nonceSz);
13713        }
13714    }
13715
13716    return ret;
13717}
13718
13719#endif /* WC_NO_RNG */
13720
13721#endif /* HAVE_AESCCM */
13722
13723#ifndef WC_NO_CONSTRUCTORS
13724
13725#define AES_NEW_INIT_PLAIN  0
13726#ifdef WOLF_PRIVATE_KEY_ID
13727#define AES_NEW_INIT_ID     1
13728#define AES_NEW_INIT_LABEL  2
13729#endif
13730
13731static Aes* _AesNew_common(void* heap, int devId, int *result_code,
13732                            int aesInitType, unsigned char* id,
13733                            int idLen, const char* label)
13734{
13735    int ret;
13736    Aes* aes = (Aes*)XMALLOC(sizeof(Aes), heap, DYNAMIC_TYPE_AES);
13737    if (aes == NULL) {
13738        ret = MEMORY_E;
13739    }
13740    else {
13741        switch (aesInitType) {
13742#ifdef WOLF_PRIVATE_KEY_ID
13743        case AES_NEW_INIT_ID:
13744            if (id == NULL || idLen == 0 || label != NULL) {
13745                ret = BAD_FUNC_ARG;
13746            }
13747            else {
13748                ret = wc_AesInit_Id(aes, id, idLen, heap, devId);
13749            }
13750            break;
13751        case AES_NEW_INIT_LABEL:
13752            if (label == NULL || id != NULL || idLen != 0) {
13753                ret = BAD_FUNC_ARG;
13754            }
13755            else {
13756                ret = wc_AesInit_Label(aes, label, heap, devId);
13757            }
13758            break;
13759#endif
13760        default:
13761            if (id != NULL || idLen != 0 || label != NULL) {
13762                ret = BAD_FUNC_ARG;
13763            }
13764            else {
13765                ret = wc_AesInit(aes, heap, devId);
13766            }
13767            break;
13768        }
13769        if (ret != 0) {
13770            XFREE(aes, heap, DYNAMIC_TYPE_AES);
13771            aes = NULL;
13772        }
13773    }
13774    (void)aesInitType;
13775    (void)id;
13776    (void)idLen;
13777    (void)label;
13778
13779    if (result_code != NULL) {
13780        *result_code = ret;
13781    }
13782
13783    return aes;
13784}
13785
13786Aes* wc_AesNew(void* heap, int devId, int *result_code)
13787{
13788    return _AesNew_common(heap, devId, result_code,
13789                          AES_NEW_INIT_PLAIN, NULL, 0, NULL);
13790}
13791
13792#ifdef WOLF_PRIVATE_KEY_ID
13793Aes* wc_AesNew_Id(unsigned char* id, int len, void* heap, int devId,
13794                   int *result_code)
13795{
13796    return _AesNew_common(heap, devId, result_code,
13797                          AES_NEW_INIT_ID, id, len, NULL);
13798}
13799
13800Aes* wc_AesNew_Label(const char* label, void* heap, int devId,
13801                      int *result_code)
13802{
13803    return _AesNew_common(heap, devId, result_code,
13804                          AES_NEW_INIT_LABEL, NULL, 0, label);
13805}
13806#endif /* WOLF_PRIVATE_KEY_ID */
13807
13808int wc_AesDelete(Aes *aes, Aes** aes_p)
13809{
13810    void* heap;
13811    if (aes == NULL)
13812        return BAD_FUNC_ARG;
13813    heap = aes->heap;
13814    wc_AesFree(aes);
13815    XFREE(aes, heap, DYNAMIC_TYPE_AES);
13816    if (aes_p != NULL)
13817        *aes_p = NULL;
13818    return 0;
13819}
13820#endif /* !WC_NO_CONSTRUCTORS */
13821
13822/* Initialize Aes */
13823int wc_AesInit(Aes* aes, void* heap, int devId)
13824{
13825    int ret = 0;
13826
13827    if (aes == NULL)
13828        return BAD_FUNC_ARG;
13829
13830    XMEMSET(aes, 0, sizeof(*aes));
13831
13832    aes->heap = heap;
13833
13834#if defined(WOLF_CRYPTO_CB) || defined(WOLFSSL_STM32U5_DHUK)
13835    aes->devId = devId;
13836    aes->devCtx = NULL;
13837#else
13838    (void)devId;
13839#endif
13840#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)
13841    ret = wolfAsync_DevCtxInit(&aes->asyncDev, WOLFSSL_ASYNC_MARKER_AES,
13842                                                        aes->heap, devId);
13843#endif /* WOLFSSL_ASYNC_CRYPT */
13844
13845#if defined(WOLFSSL_AFALG) || defined(WOLFSSL_AFALG_XILINX_AES)
13846    aes->alFd = WC_SOCK_NOTSET;
13847    aes->rdFd = WC_SOCK_NOTSET;
13848#endif
13849#if defined(WOLFSSL_DEVCRYPTO) && \
13850   (defined(WOLFSSL_DEVCRYPTO_AES) || defined(WOLFSSL_DEVCRYPTO_CBC))
13851    aes->ctx.cfd = -1;
13852#endif
13853#if defined(WOLFSSL_IMXRT_DCP)
13854    DCPAesInit(aes);
13855#endif
13856
13857#if defined(WOLFSSL_HAVE_PSA) && !defined(WOLFSSL_PSA_NO_AES)
13858    ret = wc_psa_aes_init(aes);
13859#endif
13860
13861#ifdef WC_DEBUG_CIPHER_LIFECYCLE
13862    if (ret == 0)
13863        ret = wc_debug_CipherLifecycleInit(&aes->CipherLifecycleTag, aes->heap);
13864#endif
13865
13866    return ret;
13867}
13868
13869#ifdef WOLF_PRIVATE_KEY_ID
13870int  wc_AesInit_Id(Aes* aes, unsigned char* id, int len, void* heap, int devId)
13871{
13872    int ret = 0;
13873
13874    if (aes == NULL)
13875        ret = BAD_FUNC_ARG;
13876    if (ret == 0 && (len < 0 || len > AES_MAX_ID_LEN))
13877        ret = BUFFER_E;
13878
13879    if (ret == 0)
13880        ret = wc_AesInit(aes, heap, devId);
13881    if (ret == 0) {
13882        XMEMCPY(aes->id, id, (size_t)len);
13883        aes->idLen = len;
13884        aes->labelLen = 0;
13885    }
13886
13887    return ret;
13888}
13889
13890int wc_AesInit_Label(Aes* aes, const char* label, void* heap, int devId)
13891{
13892    int ret = 0;
13893    size_t labelLen = 0;
13894
13895    if (aes == NULL || label == NULL)
13896        ret = BAD_FUNC_ARG;
13897    if (ret == 0) {
13898        labelLen = XSTRLEN(label);
13899        if (labelLen == 0 || labelLen > AES_MAX_LABEL_LEN)
13900            ret = BUFFER_E;
13901    }
13902
13903    if (ret == 0)
13904        ret = wc_AesInit(aes, heap, devId);
13905    if (ret == 0) {
13906        XMEMCPY(aes->label, label, labelLen);
13907        aes->labelLen = (int)labelLen;
13908        aes->idLen = 0;
13909    }
13910
13911    return ret;
13912}
13913#endif
13914
13915/* Free Aes resources */
13916void wc_AesFree(Aes* aes)
13917{
13918    if (aes == NULL) {
13919        return;
13920    }
13921
13922#if defined(WOLF_CRYPTO_CB) && defined(WOLF_CRYPTO_CB_FREE)
13923    #ifndef WOLF_CRYPTO_CB_FIND
13924    if (aes->devId != INVALID_DEVID)
13925    #endif
13926    {
13927        int ret = wc_CryptoCb_Free(aes->devId, WC_ALGO_TYPE_CIPHER,
13928                                   WC_CIPHER_AES, 0, aes);
13929    #ifdef WOLF_CRYPTO_CB_AES_SETKEY
13930        aes->devCtx = NULL;  /* Clear device context handle */
13931    #endif
13932        /* If callback wants standard free, it can set devId to INVALID_DEVID.
13933         * Otherwise assume the callback handled cleanup. */
13934        if (ret != WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE))
13935            return;
13936        /* fall-through when unavailable */
13937    }
13938#endif /* WOLF_CRYPTO_CB && WOLF_CRYPTO_CB_FREE */
13939
13940#ifdef WC_DEBUG_CIPHER_LIFECYCLE
13941    (void)wc_debug_CipherLifecycleFree(&aes->CipherLifecycleTag, aes->heap, 1);
13942#endif
13943
13944#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)
13945    wolfAsync_DevCtxFree(&aes->asyncDev, WOLFSSL_ASYNC_MARKER_AES);
13946#endif /* WOLFSSL_ASYNC_CRYPT */
13947#if defined(WOLFSSL_AFALG) || defined(WOLFSSL_AFALG_XILINX_AES)
13948    if (aes->rdFd > 0) { /* negative is error case */
13949        close(aes->rdFd);
13950        aes->rdFd = WC_SOCK_NOTSET;
13951    }
13952    if (aes->alFd > 0) {
13953        close(aes->alFd);
13954        aes->alFd = WC_SOCK_NOTSET;
13955    }
13956#endif /* WOLFSSL_AFALG */
13957#ifdef WOLFSSL_KCAPI_AES
13958    ForceZero((byte*)aes->devKey, AES_MAX_KEY_SIZE/WOLFSSL_BIT_SIZE);
13959    if (aes->init == 1) {
13960        kcapi_cipher_destroy(aes->handle);
13961    }
13962    aes->init = 0;
13963    aes->handle = NULL;
13964#endif
13965#if defined(WOLFSSL_DEVCRYPTO) && \
13966    (defined(WOLFSSL_DEVCRYPTO_AES) || defined(WOLFSSL_DEVCRYPTO_CBC))
13967    wc_DevCryptoFree(&aes->ctx);
13968#endif
13969#if defined(WOLF_CRYPTO_CB) || (defined(WOLFSSL_DEVCRYPTO) && \
13970    (defined(WOLFSSL_DEVCRYPTO_AES) || defined(WOLFSSL_DEVCRYPTO_CBC))) || \
13971    (defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES))
13972    ForceZero((byte*)aes->devKey, AES_MAX_KEY_SIZE/WOLFSSL_BIT_SIZE);
13973#endif
13974#if defined(WOLFSSL_IMXRT_DCP)
13975    DCPAesFree(aes);
13976#endif
13977#if defined(WOLFSSL_AESGCM_STREAM) && defined(WOLFSSL_SMALL_STACK) && \
13978    !defined(WOLFSSL_AESNI)
13979    if (aes->streamData != NULL) {
13980        ForceZero(aes->streamData, aes->streamData_sz);
13981        XFREE(aes->streamData, aes->heap, DYNAMIC_TYPE_AES);
13982        aes->streamData = NULL;
13983    }
13984#endif
13985
13986#if defined(WOLFSSL_SE050) && defined(WOLFSSL_SE050_CRYPT)
13987    if (aes->useSWCrypt == 0) {
13988        se050_aes_free(aes);
13989    }
13990#endif
13991#if defined(WOLFSSL_MICROCHIP_TA100) && defined(WOLFSSL_MICROCHIP_AESGCM)
13992    wc_Microchip_aes_free(aes);
13993#endif
13994#if defined(WOLFSSL_HAVE_PSA) && !defined(WOLFSSL_PSA_NO_AES)
13995    wc_psa_aes_free(aes);
13996#endif
13997
13998#ifdef WOLFSSL_MAXQ10XX_CRYPTO
13999    wc_MAXQ10XX_AesFree(aes);
14000#endif
14001
14002#if ((defined(WOLFSSL_RENESAS_FSPSM_TLS) || \
14003    defined(WOLFSSL_RENESAS_FSPSM_CRYPTONLY)) && \
14004    !defined(NO_WOLFSSL_RENESAS_FSPSM_AES))
14005    wc_fspsm_Aesfree(aes);
14006#endif
14007
14008    ForceZero(aes, sizeof(Aes));
14009
14010#ifdef WOLFSSL_CHECK_MEM_ZERO
14011    wc_MemZero_Check(aes, sizeof(Aes));
14012#endif
14013}
14014
14015int wc_AesGetKeySize(Aes* aes, word32* keySize)
14016{
14017    int ret = 0;
14018
14019    if (aes == NULL || keySize == NULL) {
14020        return BAD_FUNC_ARG;
14021    }
14022
14023#if defined(WOLFSSL_HAVE_PSA) && !defined(WOLFSSL_PSA_NO_AES)
14024    return wc_psa_aes_get_key_size(aes, keySize);
14025#endif
14026#if defined(WOLFSSL_CRYPTOCELL) && defined(WOLFSSL_CRYPTOCELL_AES)
14027    *keySize = aes->ctx.key.keySize;
14028    return ret;
14029#endif
14030    switch (aes->rounds) {
14031#ifdef WOLFSSL_AES_128
14032    case 10:
14033        *keySize = 16;
14034        break;
14035#endif
14036#ifdef WOLFSSL_AES_192
14037    case 12:
14038        *keySize = 24;
14039        break;
14040#endif
14041#ifdef WOLFSSL_AES_256
14042    case 14:
14043        *keySize = 32;
14044        break;
14045#endif
14046    default:
14047        *keySize = 0;
14048        ret = BAD_FUNC_ARG;
14049    }
14050
14051    return ret;
14052}
14053
14054#endif /* !WOLFSSL_TI_CRYPT */
14055
14056/* the earlier do-nothing default definitions for VECTOR_REGISTERS_{PUSH,POP}
14057 * are missed when WOLFSSL_TI_CRYPT or WOLFSSL_ARMASM.
14058 */
14059#ifndef VECTOR_REGISTERS_PUSH
14060    #define VECTOR_REGISTERS_PUSH { WC_DO_NOTHING
14061#endif
14062#ifndef VECTOR_REGISTERS_POP
14063    #define VECTOR_REGISTERS_POP } WC_DO_NOTHING
14064#endif
14065
14066#ifdef HAVE_AES_ECB
14067#if defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES) && \
14068        !defined(WOLFSSL_QNX_CAAM)
14069    /* implemented in wolfcrypt/src/port/caam/caam_aes.c */
14070
14071#elif defined(WOLFSSL_AFALG)
14072    /* implemented in wolfcrypt/src/port/af_alg/afalg_aes.c */
14073
14074#elif defined(WOLFSSL_DEVCRYPTO_AES)
14075    /* implemented in wolfcrypt/src/port/devcrypt/devcrypto_aes.c */
14076
14077#elif defined(WOLFSSL_RISCV_ASM)
14078    /* implemented in wolfcrypt/src/port/riscv/riscv-64-aes.c */
14079
14080#elif defined(WOLFSSL_NXP_HASHCRYPT_AES)
14081    /* implemented in wolfcrypt/src/port/nxp/hashcrypt_port.c */
14082
14083#elif defined(WOLFSSL_SILABS_SE_ACCEL)
14084    /* implemented in wolfcrypt/src/port/silabs/silabs_aes.c */
14085
14086#elif defined(MAX3266X_AES)
14087
14088int wc_AesEcbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
14089{
14090    int status;
14091    word32 keySize;
14092
14093    if ((in == NULL) || (out == NULL) || (aes == NULL))
14094        return BAD_FUNC_ARG;
14095
14096    status = wc_AesGetKeySize(aes, &keySize);
14097    if (status != 0) {
14098        return status;
14099    }
14100
14101    status = wc_MXC_TPU_AesEncrypt(in, (byte*)aes->reg, (byte*)aes->key,
14102                                        MXC_TPU_MODE_ECB, sz, out, keySize);
14103
14104    return status;
14105}
14106
14107#ifdef HAVE_AES_DECRYPT
14108int wc_AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
14109{
14110    int status;
14111    word32 keySize;
14112
14113    if ((in == NULL) || (out == NULL) || (aes == NULL))
14114        return BAD_FUNC_ARG;
14115
14116    status = wc_AesGetKeySize(aes, &keySize);
14117    if (status != 0) {
14118        return status;
14119    }
14120
14121    status = wc_MXC_TPU_AesDecrypt(in, (byte*)aes->reg, (byte*)aes->key,
14122                                        MXC_TPU_MODE_ECB, sz, out, keySize);
14123
14124    return status;
14125}
14126#endif /* HAVE_AES_DECRYPT */
14127
14128#elif defined(WOLFSSL_SCE) && !defined(WOLFSSL_SCE_NO_AES)
14129
14130/* Software AES - ECB */
14131int wc_AesEcbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
14132{
14133    if ((in == NULL) || (out == NULL) || (aes == NULL))
14134        return BAD_FUNC_ARG;
14135
14136    return AES_ECB_encrypt(aes, in, out, sz);
14137}
14138
14139
14140int wc_AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
14141{
14142    if ((in == NULL) || (out == NULL) || (aes == NULL))
14143        return BAD_FUNC_ARG;
14144
14145    return AES_ECB_decrypt(aes, in, out, sz);
14146}
14147
14148#elif defined(WOLFSSL_PSOC6_CRYPTO)
14149
14150int wc_AesEcbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
14151{
14152    if ((in == NULL) || (out == NULL) || (aes == NULL))
14153        return BAD_FUNC_ARG;
14154
14155    return wc_Psoc6_Aes_EcbEncrypt(aes, out, in, sz);
14156}
14157
14158#define _AesEcbEncrypt(aes, out, in, sz) wc_AesEcbEncrypt(aes, out, in, sz)
14159
14160#ifdef HAVE_AES_DECRYPT
14161int wc_AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
14162{
14163    if ((in == NULL) || (out == NULL) || (aes == NULL))
14164        return BAD_FUNC_ARG;
14165
14166    return wc_Psoc6_Aes_EcbDecrypt(aes, out, in, sz);
14167}
14168
14169#define _AesEcbDecrypt(aes, out, in, sz) wc_AesEcbDecrypt(aes, out, in, sz)
14170#endif /* HAVE_AES_DECRYPT */
14171
14172#else
14173
14174/* Software AES - ECB */
14175static WARN_UNUSED_RESULT int _AesEcbEncrypt(
14176    Aes* aes, byte* out, const byte* in, word32 sz)
14177{
14178    int ret = 0;
14179
14180#ifdef WOLF_CRYPTO_CB
14181    #ifndef WOLF_CRYPTO_CB_FIND
14182    if (aes->devId != INVALID_DEVID)
14183    #endif
14184    {
14185        ret = wc_CryptoCb_AesEcbEncrypt(aes, out, in, sz);
14186        if (ret != WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE))
14187            return ret;
14188        ret = 0;
14189        /* fall-through when unavailable */
14190    }
14191#endif
14192#ifdef WOLF_CRYPTO_CB_ONLY_AES
14193    /* No software fallback: the per-block loop below would only re-invoke
14194     * cryptocb ECB and propagate UNAVAILABLE; short-circuit instead. */
14195    return NO_VALID_DEVID;
14196#endif
14197#ifdef WOLFSSL_IMXRT_DCP
14198    if (aes->keylen == 16)
14199        return DCPAesEcbEncrypt(aes, out, in, sz);
14200#endif
14201
14202    VECTOR_REGISTERS_PUSH;
14203
14204#if !defined(__aarch64__) && defined(WOLFSSL_ARMASM)
14205#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
14206    AES_encrypt_blocks_AARCH32(in, out, sz, (byte*)aes->key, (int)aes->rounds);
14207#else
14208    AES_ECB_encrypt(in, out, sz, (const unsigned char*)aes->key, aes->rounds);
14209#endif
14210#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM)
14211#if !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
14212    if (aes->use_aes_hw_crypto) {
14213        AES_encrypt_blocks_AARCH64(in, out, sz, (byte*)aes->key,
14214            (int)aes->rounds);
14215    }
14216    else
14217#endif
14218#if !defined(WOLFSSL_ARMASM_NO_NEON)
14219#ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
14220    if (sz >= 32)
14221#endif
14222    {
14223        AES_ECB_encrypt_NEON(in, out, sz, (const unsigned char*)aes->key,
14224            aes->rounds);
14225    }
14226#ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
14227    else
14228#endif
14229#endif
14230#ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
14231    {
14232        AES_ECB_encrypt(in, out, sz, (const unsigned char*)aes->key,
14233            aes->rounds);
14234    }
14235#endif
14236#else
14237#ifdef WOLFSSL_AESNI
14238    if (aes->use_aesni) {
14239        AES_ECB_encrypt_AESNI(in, out, sz, (byte*)aes->key, (int)aes->rounds);
14240    }
14241    else
14242#endif
14243    {
14244#if defined(NEED_AES_TABLES)
14245        AesEncryptBlocks_C(aes, in, out, sz);
14246#else
14247        word32 i;
14248#ifdef WC_AES_HAVE_PREFETCH_ARG
14249        int did_prefetches = 0;
14250#endif
14251
14252        for (i = 0; i < sz; i += WC_AES_BLOCK_SIZE) {
14253            ret = AesEncrypt_preFetchOpt(aes, in, out, &did_prefetches);
14254            if (ret != 0)
14255                break;
14256            in += WC_AES_BLOCK_SIZE;
14257            out += WC_AES_BLOCK_SIZE;
14258        }
14259#endif
14260    }
14261#endif
14262
14263    VECTOR_REGISTERS_POP;
14264
14265    return ret;
14266}
14267
14268#ifdef HAVE_AES_DECRYPT
14269static WARN_UNUSED_RESULT int _AesEcbDecrypt(
14270    Aes* aes, byte* out, const byte* in, word32 sz)
14271{
14272    int ret = 0;
14273
14274#ifdef WOLF_CRYPTO_CB
14275    #ifndef WOLF_CRYPTO_CB_FIND
14276    if (aes->devId != INVALID_DEVID)
14277    #endif
14278    {
14279        ret = wc_CryptoCb_AesEcbDecrypt(aes, out, in, sz);
14280        if (ret != WC_NO_ERR_TRACE(CRYPTOCB_UNAVAILABLE))
14281            return ret;
14282        ret = 0;
14283        /* fall-through when unavailable */
14284    }
14285#endif
14286#ifdef WOLF_CRYPTO_CB_ONLY_AES
14287    return NO_VALID_DEVID;
14288#endif
14289#ifdef WOLFSSL_IMXRT_DCP
14290    if (aes->keylen == 16)
14291        return DCPAesEcbDecrypt(aes, out, in, sz);
14292#endif
14293
14294    VECTOR_REGISTERS_PUSH;
14295
14296#if !defined(__aarch64__) && defined(WOLFSSL_ARMASM)
14297#ifndef WOLFSSL_ARMASM_NO_HW_CRYPTO
14298    AES_decrypt_blocks_AARCH32(in, out, sz, (byte*)aes->key, (int)aes->rounds);
14299#else
14300    AES_ECB_decrypt(in, out, sz, (const unsigned char*)aes->key, aes->rounds);
14301#endif
14302#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM)
14303#if !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
14304    if (aes->use_aes_hw_crypto) {
14305        AES_decrypt_blocks_AARCH64(in, out, sz, (byte*)aes->key,
14306            (int)aes->rounds);
14307    }
14308    else
14309#endif
14310#if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON)
14311#ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
14312    if (sz >= 64)
14313#endif
14314    {
14315        AES_ECB_decrypt_NEON(in, out, sz, (const unsigned char*)aes->key,
14316            aes->rounds);
14317    }
14318#ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
14319    else
14320#endif
14321#endif
14322#ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
14323    {
14324        AES_ECB_decrypt(in, out, sz, (const unsigned char*)aes->key,
14325            aes->rounds);
14326    }
14327#endif
14328#else
14329#ifdef WOLFSSL_AESNI
14330    if (aes->use_aesni) {
14331        AES_ECB_decrypt_AESNI(in, out, sz, (byte*)aes->key, (int)aes->rounds);
14332    }
14333    else
14334#endif
14335    {
14336#if defined(NEED_AES_TABLES)
14337        AesDecryptBlocks_C(aes, in, out, sz);
14338#else
14339        word32 i;
14340
14341        for (i = 0; i < sz; i += WC_AES_BLOCK_SIZE) {
14342            ret = wc_AesDecryptDirect(aes, out, in);
14343            if (ret != 0)
14344                break;
14345            in += WC_AES_BLOCK_SIZE;
14346            out += WC_AES_BLOCK_SIZE;
14347        }
14348#endif
14349    }
14350#endif
14351
14352    VECTOR_REGISTERS_POP;
14353
14354    return ret;
14355}
14356#endif
14357
14358int wc_AesEcbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
14359{
14360    if ((in == NULL) || (out == NULL) || (aes == NULL))
14361      return BAD_FUNC_ARG;
14362    if ((sz % WC_AES_BLOCK_SIZE) != 0) {
14363        return BAD_LENGTH_E;
14364    }
14365
14366    return _AesEcbEncrypt(aes, out, in, sz);
14367}
14368
14369#ifdef HAVE_AES_DECRYPT
14370int wc_AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
14371{
14372    if ((in == NULL) || (out == NULL) || (aes == NULL))
14373      return BAD_FUNC_ARG;
14374    if ((sz % WC_AES_BLOCK_SIZE) != 0) {
14375        return BAD_LENGTH_E;
14376    }
14377
14378    return _AesEcbDecrypt(aes, out, in, sz);
14379}
14380#endif /* HAVE_AES_DECRYPT */
14381#endif
14382#endif /* HAVE_AES_ECB */
14383
14384#if defined(WOLFSSL_AES_CFB)
14385
14386#if defined(WOLFSSL_NXP_HASHCRYPT_AES)
14387    /* implemented in wolfcrypt/src/port/nxp/hashcrypt_port.c */
14388
14389#elif defined(WOLFSSL_PSOC6_CRYPTO)
14390
14391int wc_AesCfbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
14392{
14393    return wc_Psoc6_Aes_CfbEncrypt(aes, out, in, sz);
14394}
14395
14396#ifdef HAVE_AES_DECRYPT
14397int wc_AesCfbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
14398{
14399    return wc_Psoc6_Aes_CfbDecrypt(aes, out, in, sz);
14400}
14401#endif /* HAVE_AES_DECRYPT */
14402
14403#else
14404/* Feedback AES mode
14405 *
14406 * aes structure holding key to use for encryption
14407 * out buffer to hold result of encryption (must be at least as large as input
14408 *     buffer)
14409 * in  buffer to encrypt
14410 * sz  size of input buffer
14411 * mode flag to specify AES mode
14412 *
14413 * returns 0 on success and negative error values on failure
14414 */
14415/* Software AES - CFB Encrypt */
14416static WARN_UNUSED_RESULT int AesCfbEncrypt_C(Aes* aes, byte* out,
14417    const byte* in, word32 sz)
14418{
14419    int ret = 0;
14420    word32 processed;
14421#ifdef WC_AES_HAVE_PREFETCH_ARG
14422    int did_prefetches = 0;
14423#endif
14424
14425    if ((aes == NULL) || (out == NULL) || (in == NULL)) {
14426        return BAD_FUNC_ARG;
14427    }
14428    if (sz == 0) {
14429        return 0;
14430    }
14431
14432    if (aes->left > 0) {
14433        /* consume any unused bytes left in aes->tmp */
14434        processed = min(aes->left, sz);
14435        xorbufout(out, in, (byte*)aes->tmp + WC_AES_BLOCK_SIZE - aes->left,
14436            processed);
14437        XMEMCPY((byte*)aes->reg + WC_AES_BLOCK_SIZE - aes->left, out,
14438            processed);
14439        aes->left -= processed;
14440        out += processed;
14441        in += processed;
14442        sz -= processed;
14443    }
14444
14445    VECTOR_REGISTERS_PUSH;
14446
14447    while (sz >= WC_AES_BLOCK_SIZE) {
14448        ret = AesEncrypt_preFetchOpt(aes, (byte*)aes->reg, (byte*)aes->reg,
14449                                        &did_prefetches);
14450        if (ret != 0) {
14451            break;
14452        }
14453        xorbuf((byte*)aes->reg, in, WC_AES_BLOCK_SIZE);
14454        XMEMCPY(out, aes->reg, WC_AES_BLOCK_SIZE);
14455        out += WC_AES_BLOCK_SIZE;
14456        in  += WC_AES_BLOCK_SIZE;
14457        sz  -= WC_AES_BLOCK_SIZE;
14458    }
14459
14460    /* encrypt left over data */
14461    if ((ret == 0) && sz) {
14462        ret = AesEncrypt_preFetchOpt(aes, (byte*)aes->reg, (byte*)aes->tmp,
14463                                     &did_prefetches);
14464        if (ret == 0) {
14465            xorbufout(out, in, aes->tmp, sz);
14466            XMEMCPY(aes->reg, out, sz);
14467            aes->left = WC_AES_BLOCK_SIZE - sz;
14468        }
14469    }
14470
14471    VECTOR_REGISTERS_POP;
14472
14473    return ret;
14474}
14475
14476
14477#if defined(HAVE_AES_DECRYPT)
14478/* CFB 128
14479 *
14480 * aes structure holding key to use for decryption
14481 * out buffer to hold result of decryption (must be at least as large as input
14482 *     buffer)
14483 * in  buffer to decrypt
14484 * sz  size of input buffer
14485 *
14486 * returns 0 on success and negative error values on failure
14487 */
14488/* Software AES - CFB Decrypt */
14489static WARN_UNUSED_RESULT int AesCfbDecrypt_C(Aes* aes, byte* out,
14490    const byte* in, word32 sz, byte mode)
14491{
14492    int ret = 0;
14493    word32 processed;
14494#ifdef WC_AES_HAVE_PREFETCH_ARG
14495    int did_prefetches = 0;
14496#endif
14497
14498    (void)mode;
14499
14500    if ((aes == NULL) || (out == NULL) || (in == NULL)) {
14501        return BAD_FUNC_ARG;
14502    }
14503    if (sz == 0) {
14504        return 0;
14505    }
14506
14507    if (aes->left > 0) {
14508        /* consume any unused bytes left in aes->tmp */
14509        processed = min(aes->left, sz);
14510        /* copy input over to aes->reg */
14511        XMEMCPY((byte*)aes->reg + WC_AES_BLOCK_SIZE - aes->left, in, processed);
14512        xorbufout(out, in, (byte*)aes->tmp + WC_AES_BLOCK_SIZE - aes->left,
14513            processed);
14514        aes->left -= processed;
14515        out += processed;
14516        in += processed;
14517        sz -= processed;
14518    }
14519
14520    VECTOR_REGISTERS_PUSH;
14521
14522    #if !defined(WOLFSSL_SMALL_STACK) && defined(HAVE_AES_ECB) && \
14523        !defined(WOLFSSL_PIC32MZ_CRYPT) && \
14524        (defined(USE_INTEL_SPEEDUP) || defined(WOLFSSL_ARMASM))
14525    {
14526        ALIGN16 byte tmp[4 * WC_AES_BLOCK_SIZE];
14527        while (sz >= 4 * WC_AES_BLOCK_SIZE) {
14528            XMEMCPY(tmp, aes->reg, WC_AES_BLOCK_SIZE);
14529            XMEMCPY(tmp + WC_AES_BLOCK_SIZE, in, 3 * WC_AES_BLOCK_SIZE);
14530            XMEMCPY(aes->reg, in + 3 * WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
14531            ret = wc_AesEcbEncrypt(aes, tmp, tmp, 4 * WC_AES_BLOCK_SIZE);
14532            if (ret != 0) {
14533                break;
14534            }
14535            xorbufout(out, in, tmp, 4 * WC_AES_BLOCK_SIZE);
14536            out += 4 * WC_AES_BLOCK_SIZE;
14537            in  += 4 * WC_AES_BLOCK_SIZE;
14538            sz  -= 4 * WC_AES_BLOCK_SIZE;
14539        }
14540    }
14541    #endif
14542    while (sz >= WC_AES_BLOCK_SIZE) {
14543        ret = AesEncrypt_preFetchOpt(aes, (byte*)aes->reg, (byte*)aes->tmp,
14544                                        &did_prefetches);
14545        if (ret != 0) {
14546            break;
14547        }
14548        XMEMCPY((byte*)aes->reg, in, WC_AES_BLOCK_SIZE);
14549        xorbufout(out, in, (byte*)aes->tmp, WC_AES_BLOCK_SIZE);
14550        out += WC_AES_BLOCK_SIZE;
14551        in  += WC_AES_BLOCK_SIZE;
14552        sz  -= WC_AES_BLOCK_SIZE;
14553    }
14554
14555    /* decrypt left over data */
14556    if ((ret == 0) && sz) {
14557        ret = AesEncrypt_preFetchOpt(aes, (byte*)aes->reg, (byte*)aes->tmp,
14558                                        &did_prefetches);
14559        if (ret == 0) {
14560            XMEMCPY(aes->reg, in, sz);
14561            xorbufout(out, in, aes->tmp, sz);
14562            aes->left = WC_AES_BLOCK_SIZE - sz;
14563        }
14564    }
14565
14566    VECTOR_REGISTERS_POP;
14567
14568    return ret;
14569}
14570#endif /* HAVE_AES_DECRYPT */
14571
14572/* CFB 128
14573 *
14574 * aes structure holding key to use for encryption
14575 * out buffer to hold result of encryption (must be at least as large as input
14576 *     buffer)
14577 * in  buffer to encrypt
14578 * sz  size of input buffer
14579 *
14580 * returns 0 on success and negative error values on failure
14581 */
14582/* Software AES - CFB Encrypt */
14583int wc_AesCfbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
14584{
14585    return AesCfbEncrypt_C(aes, out, in, sz);
14586}
14587
14588
14589#ifdef HAVE_AES_DECRYPT
14590/* CFB 128
14591 *
14592 * aes structure holding key to use for decryption
14593 * out buffer to hold result of decryption (must be at least as large as input
14594 *     buffer)
14595 * in  buffer to decrypt
14596 * sz  size of input buffer
14597 *
14598 * returns 0 on success and negative error values on failure
14599 */
14600/* Software AES - CFB Decrypt */
14601int wc_AesCfbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
14602{
14603    return AesCfbDecrypt_C(aes, out, in, sz, AES_CFB_MODE);
14604}
14605#endif /* HAVE_AES_DECRYPT */
14606#endif /* WOLFSSL_PSOC6_CRYPTO */
14607
14608#ifndef WOLFSSL_NO_AES_CFB_1_8
14609/* shift the whole WC_AES_BLOCK_SIZE array left by 8 or 1 bits */
14610static void shiftLeftArray(byte* ary, byte shift)
14611{
14612    int i;
14613
14614    if (shift == WOLFSSL_BIT_SIZE) {
14615        /* shifting over by 8 bits */
14616        for (i = 0; i < WC_AES_BLOCK_SIZE - 1; i++) {
14617            ary[i] = ary[i+1];
14618        }
14619        ary[i] = 0;
14620    }
14621    else {
14622        /* shifting over by 7 or less bits */
14623        for (i = 0; i < WC_AES_BLOCK_SIZE - 1; i++) {
14624            byte carry = (byte)(ary[i+1] & (0XFF << (WOLFSSL_BIT_SIZE - shift)));
14625            carry = (byte)(carry >> (WOLFSSL_BIT_SIZE - shift));
14626            ary[i] = (byte)((ary[i] << shift) + carry);
14627        }
14628        ary[i] = (byte)(ary[i] << shift);
14629    }
14630}
14631
14632
14633/* returns 0 on success and negative values on failure */
14634static WARN_UNUSED_RESULT int wc_AesFeedbackCFB8(
14635    Aes* aes, byte* out, const byte* in, word32 sz, byte dir)
14636{
14637    byte *pt;
14638    int ret = 0;
14639#ifdef WC_AES_HAVE_PREFETCH_ARG
14640    int did_prefetches = 0;
14641#endif
14642
14643    if (aes == NULL || out == NULL || in == NULL) {
14644        return BAD_FUNC_ARG;
14645    }
14646
14647    if (sz == 0) {
14648        return 0;
14649    }
14650
14651    VECTOR_REGISTERS_PUSH;
14652
14653    while (sz > 0) {
14654        ret = AesEncrypt_preFetchOpt(aes, (byte*)aes->reg, (byte*)aes->tmp,
14655                                        &did_prefetches);
14656        if (ret != 0)
14657            break;
14658        if (dir == AES_DECRYPTION) {
14659            pt = (byte*)aes->reg;
14660
14661            /* LSB + CAT */
14662            shiftLeftArray(pt, WOLFSSL_BIT_SIZE);
14663            pt[WC_AES_BLOCK_SIZE - 1] = in[0];
14664        }
14665
14666        /* MSB + XOR */
14667    #ifdef BIG_ENDIAN_ORDER
14668        ByteReverseWords(aes->tmp, aes->tmp, WC_AES_BLOCK_SIZE);
14669    #endif
14670        out[0] = (byte)(aes->tmp[0] ^ in[0]);
14671        if (dir == AES_ENCRYPTION) {
14672            pt = (byte*)aes->reg;
14673
14674            /* LSB + CAT */
14675            shiftLeftArray(pt, WOLFSSL_BIT_SIZE);
14676            pt[WC_AES_BLOCK_SIZE - 1] = out[0];
14677        }
14678
14679        out += 1;
14680        in  += 1;
14681        sz  -= 1;
14682    }
14683
14684    VECTOR_REGISTERS_POP;
14685
14686    return ret;
14687}
14688
14689
14690/* returns 0 on success and negative values on failure */
14691static WARN_UNUSED_RESULT int wc_AesFeedbackCFB1(
14692    Aes* aes, byte* out, const byte* in, word32 sz, byte dir)
14693{
14694    byte tmp;
14695    byte cur = 0; /* hold current work in order to handle inline in=out */
14696    byte* pt;
14697    int bit = 7;
14698    int ret = 0;
14699#ifdef WC_AES_HAVE_PREFETCH_ARG
14700    int did_prefetches = 0;
14701#endif
14702
14703    if (aes == NULL || out == NULL || in == NULL) {
14704        return BAD_FUNC_ARG;
14705    }
14706
14707    if (sz == 0) {
14708        return 0;
14709    }
14710
14711    VECTOR_REGISTERS_PUSH;
14712
14713    while (sz > 0) {
14714        ret = AesEncrypt_preFetchOpt(aes, (byte*)aes->reg, (byte*)aes->tmp,
14715                                        &did_prefetches);
14716        if (ret != 0)
14717            break;
14718        if (dir == AES_DECRYPTION) {
14719            pt = (byte*)aes->reg;
14720
14721            /* LSB + CAT */
14722            tmp = (byte)((0X01U << bit) & in[0]);
14723            tmp = (byte)(tmp >> bit);
14724            tmp &= 0x01;
14725            shiftLeftArray((byte*)aes->reg, 1);
14726            pt[WC_AES_BLOCK_SIZE - 1] |= tmp;
14727        }
14728
14729        /* MSB  + XOR */
14730        tmp = (byte)((0X01U << bit) & in[0]);
14731        pt = (byte*)aes->tmp;
14732        tmp = (byte)((pt[0] >> 7) ^ (tmp >> bit));
14733        tmp &= 0x01;
14734        cur = (byte)(cur | (tmp << bit));
14735
14736
14737        if (dir == AES_ENCRYPTION) {
14738            pt = (byte*)aes->reg;
14739
14740            /* LSB + CAT */
14741            shiftLeftArray((byte*)aes->reg, 1);
14742            pt[WC_AES_BLOCK_SIZE - 1] |= tmp;
14743        }
14744
14745        bit--;
14746        if (bit < 0) {
14747            out[0] = cur;
14748            out += 1;
14749            in  += 1;
14750            sz  -= 1;
14751            bit = 7U;
14752            cur = 0;
14753        }
14754        else {
14755            sz -= 1;
14756        }
14757    }
14758
14759    if (ret == 0) {
14760        if (bit >= 0 && bit < 7) {
14761            out[0] = cur;
14762        }
14763    }
14764
14765    VECTOR_REGISTERS_POP;
14766
14767    return ret;
14768}
14769
14770
14771/* CFB 1
14772 *
14773 * aes structure holding key to use for encryption
14774 * out buffer to hold result of encryption (must be at least as large as input
14775 *     buffer)
14776 * in  buffer to encrypt (packed to left, i.e. 101 is 0x90)
14777 * sz  size of input buffer in bits (0x1 would be size of 1 and 0xFF size of 8)
14778 *
14779 * returns 0 on success and negative values on failure
14780 */
14781int wc_AesCfb1Encrypt(Aes* aes, byte* out, const byte* in, word32 sz)
14782{
14783    return wc_AesFeedbackCFB1(aes, out, in, sz, AES_ENCRYPTION);
14784}
14785
14786
14787/* CFB 8
14788 *
14789 * aes structure holding key to use for encryption
14790 * out buffer to hold result of encryption (must be at least as large as input
14791 *     buffer)
14792 * in  buffer to encrypt
14793 * sz  size of input buffer
14794 *
14795 * returns 0 on success and negative values on failure
14796 */
14797int wc_AesCfb8Encrypt(Aes* aes, byte* out, const byte* in, word32 sz)
14798{
14799    return wc_AesFeedbackCFB8(aes, out, in, sz, AES_ENCRYPTION);
14800}
14801#ifdef HAVE_AES_DECRYPT
14802
14803/* CFB 1
14804 *
14805 * aes structure holding key to use for encryption
14806 * out buffer to hold result of encryption (must be at least as large as input
14807 *     buffer)
14808 * in  buffer to encrypt
14809 * sz  size of input buffer in bits (0x1 would be size of 1 and 0xFF size of 8)
14810 *
14811 * returns 0 on success and negative values on failure
14812 */
14813int wc_AesCfb1Decrypt(Aes* aes, byte* out, const byte* in, word32 sz)
14814{
14815    return wc_AesFeedbackCFB1(aes, out, in, sz, AES_DECRYPTION);
14816}
14817
14818
14819/* CFB 8
14820 *
14821 * aes structure holding key to use for encryption
14822 * out buffer to hold result of encryption (must be at least as large as input
14823 *     buffer)
14824 * in  buffer to encrypt
14825 * sz  size of input buffer
14826 *
14827 * returns 0 on success and negative values on failure
14828 */
14829int wc_AesCfb8Decrypt(Aes* aes, byte* out, const byte* in, word32 sz)
14830{
14831    return wc_AesFeedbackCFB8(aes, out, in, sz, AES_DECRYPTION);
14832}
14833#endif /* HAVE_AES_DECRYPT */
14834#endif /* !WOLFSSL_NO_AES_CFB_1_8 */
14835#endif /* WOLFSSL_AES_CFB */
14836
14837#ifdef WOLFSSL_AES_OFB
14838#ifdef WOLFSSL_NXP_HASHCRYPT_AES
14839    /* implemented in wolfcrypt/src/port/nxp/hashcrypt_port.c */
14840
14841#else /* software */
14842/* OFB AES mode
14843 *
14844 * aes structure holding key to use for encryption
14845 * out buffer to hold result of encryption (must be at least as large as input
14846 *     buffer)
14847 * in  buffer to encrypt
14848 * sz  size of input buffer
14849 *
14850 * returns 0 on success and negative error values on failure
14851 */
14852/* Software AES - OFB Encrypt/Decrypt */
14853static WARN_UNUSED_RESULT int AesOfbCrypt_C(Aes* aes, byte* out, const byte* in,
14854    word32 sz)
14855{
14856    int ret = 0;
14857    word32 processed;
14858#ifdef WC_AES_HAVE_PREFETCH_ARG
14859    int did_prefetches = 0;
14860#endif
14861
14862    if ((aes == NULL) || (out == NULL) || (in == NULL)) {
14863        return BAD_FUNC_ARG;
14864    }
14865    if (sz == 0) {
14866        return 0;
14867    }
14868
14869    if (aes->left > 0) {
14870        /* consume any unused bytes left in aes->tmp */
14871        processed = min(aes->left, sz);
14872        xorbufout(out, in, (byte*)aes->tmp + WC_AES_BLOCK_SIZE - aes->left,
14873            processed);
14874        aes->left -= processed;
14875        out += processed;
14876        in += processed;
14877        sz -= processed;
14878    }
14879
14880    VECTOR_REGISTERS_PUSH;
14881
14882    while (sz >= WC_AES_BLOCK_SIZE) {
14883        ret = AesEncrypt_preFetchOpt(aes, (byte*)aes->reg, (byte*)aes->reg,
14884                                        &did_prefetches);
14885        if (ret != 0) {
14886            break;
14887        }
14888        xorbufout(out, in, (byte*)aes->reg, WC_AES_BLOCK_SIZE);
14889        out += WC_AES_BLOCK_SIZE;
14890        in  += WC_AES_BLOCK_SIZE;
14891        sz  -= WC_AES_BLOCK_SIZE;
14892    }
14893
14894    /* encrypt left over data */
14895    if ((ret == 0) && sz) {
14896        ret = AesEncrypt_preFetchOpt(aes, (byte*)aes->reg, (byte*)aes->tmp,
14897                                        &did_prefetches);
14898        if (ret == 0) {
14899            XMEMCPY(aes->reg, aes->tmp, WC_AES_BLOCK_SIZE);
14900            xorbufout(out, in, aes->tmp, sz);
14901            aes->left = WC_AES_BLOCK_SIZE - sz;
14902        }
14903    }
14904
14905    VECTOR_REGISTERS_POP;
14906
14907    return ret;
14908}
14909
14910/* OFB
14911 *
14912 * aes structure holding key to use for encryption
14913 * out buffer to hold result of encryption (must be at least as large as input
14914 *     buffer)
14915 * in  buffer to encrypt
14916 * sz  size of input buffer
14917 *
14918 * returns 0 on success and negative error values on failure
14919 */
14920/* Software AES - OFB Encrypt */
14921int wc_AesOfbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
14922{
14923    return AesOfbCrypt_C(aes, out, in, sz);
14924}
14925
14926
14927#ifdef HAVE_AES_DECRYPT
14928/* OFB
14929 *
14930 * aes structure holding key to use for decryption
14931 * out buffer to hold result of decryption (must be at least as large as input
14932 *     buffer)
14933 * in  buffer to decrypt
14934 * sz  size of input buffer
14935 *
14936 * returns 0 on success and negative error values on failure
14937 */
14938/* Software AES - OFB Decrypt */
14939int wc_AesOfbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
14940{
14941    return AesOfbCrypt_C(aes, out, in, sz);
14942}
14943#endif /* HAVE_AES_DECRYPT */
14944#endif /* software */
14945#endif /* WOLFSSL_AES_OFB */
14946
14947
14948#ifdef HAVE_AES_KEYWRAP
14949
14950/* Initialize key wrap counter with value */
14951static WC_INLINE void InitKeyWrapCounter(byte* inOutCtr, word32 value)
14952{
14953    word32 i;
14954    word32 bytes;
14955
14956    bytes = sizeof(word32);
14957    for (i = 0; i < sizeof(word32); i++) {
14958        inOutCtr[i+sizeof(word32)] = (byte)(value >> ((bytes - 1) * 8));
14959        bytes--;
14960    }
14961}
14962
14963/* Increment key wrap counter */
14964static WC_INLINE void IncrementKeyWrapCounter(byte* inOutCtr)
14965{
14966    int i;
14967
14968    /* in network byte order so start at end and work back */
14969    for (i = KEYWRAP_BLOCK_SIZE - 1; i >= 0; i--) {
14970        if (++inOutCtr[i])  /* we're done unless we overflow */
14971            return;
14972    }
14973}
14974
14975/* Decrement key wrap counter */
14976static WC_INLINE void DecrementKeyWrapCounter(byte* inOutCtr)
14977{
14978    int i;
14979
14980    for (i = KEYWRAP_BLOCK_SIZE - 1; i >= 0; i--) {
14981        if (--inOutCtr[i] != 0xFF)  /* we're done unless we underflow */
14982            return;
14983    }
14984}
14985
14986int wc_AesKeyWrap_ex(Aes *aes, const byte* in, word32 inSz, byte* out,
14987        word32 outSz, const byte* iv)
14988{
14989    word32 i;
14990    byte* r;
14991    int j;
14992    int ret = 0;
14993
14994    byte t[KEYWRAP_BLOCK_SIZE];
14995    byte tmp[WC_AES_BLOCK_SIZE];
14996
14997    /* n must be at least 2 64-bit blocks, output size is (n + 1) 8 bytes (64-bit) */
14998    if (aes == NULL || in  == NULL || inSz < 2*KEYWRAP_BLOCK_SIZE ||
14999        out == NULL || outSz < (inSz + KEYWRAP_BLOCK_SIZE))
15000        return BAD_FUNC_ARG;
15001
15002    /* input must be multiple of 64-bits */
15003    if (inSz % KEYWRAP_BLOCK_SIZE != 0)
15004        return BAD_FUNC_ARG;
15005
15006    r = out + 8;
15007    XMEMCPY(r, in, inSz);
15008    XMEMSET(t, 0, sizeof(t));
15009
15010    /* user IV is optional */
15011    if (iv == NULL) {
15012        XMEMSET(tmp, 0xA6, KEYWRAP_BLOCK_SIZE);
15013    } else {
15014        XMEMCPY(tmp, iv, KEYWRAP_BLOCK_SIZE);
15015    }
15016
15017    VECTOR_REGISTERS_PUSH;
15018
15019    for (j = 0; j <= 5; j++) {
15020        for (i = 1; i <= inSz / KEYWRAP_BLOCK_SIZE; i++) {
15021            /* load R[i] */
15022            XMEMCPY(tmp + KEYWRAP_BLOCK_SIZE, r, KEYWRAP_BLOCK_SIZE);
15023
15024            ret = wc_AesEncryptDirect(aes, tmp, tmp);
15025            if (ret != 0)
15026                break;
15027
15028            /* calculate new A */
15029            IncrementKeyWrapCounter(t);
15030            xorbuf(tmp, t, KEYWRAP_BLOCK_SIZE);
15031
15032            /* save R[i] */
15033            XMEMCPY(r, tmp + KEYWRAP_BLOCK_SIZE, KEYWRAP_BLOCK_SIZE);
15034            r += KEYWRAP_BLOCK_SIZE;
15035        }
15036        if (ret != 0)
15037            break;
15038        r = out + KEYWRAP_BLOCK_SIZE;
15039    }
15040
15041    VECTOR_REGISTERS_POP;
15042
15043    if (ret != 0)
15044        return ret;
15045
15046    /* C[0] = A */
15047    XMEMCPY(out, tmp, KEYWRAP_BLOCK_SIZE);
15048
15049    return (int)(inSz + KEYWRAP_BLOCK_SIZE);
15050}
15051
15052/* perform AES key wrap (RFC3394), return out sz on success, negative on err */
15053int wc_AesKeyWrap(const byte* key, word32 keySz, const byte* in, word32 inSz,
15054                  byte* out, word32 outSz, const byte* iv)
15055{
15056    WC_DECLARE_VAR(aes, Aes, 1, 0);
15057    int ret;
15058
15059    if (key == NULL)
15060        return BAD_FUNC_ARG;
15061
15062#ifdef WOLFSSL_SMALL_STACK
15063    if ((aes = (Aes *)XMALLOC(sizeof *aes, NULL,
15064                              DYNAMIC_TYPE_AES)) == NULL)
15065        return MEMORY_E;
15066#endif
15067
15068    ret = wc_AesInit(aes, NULL, INVALID_DEVID);
15069    if (ret != 0)
15070        goto out;
15071
15072    ret = wc_AesSetKey(aes, key, keySz, NULL, AES_ENCRYPTION);
15073    if (ret != 0) {
15074        wc_AesFree(aes);
15075        goto out;
15076    }
15077
15078    ret = wc_AesKeyWrap_ex(aes, in, inSz, out, outSz, iv);
15079
15080    wc_AesFree(aes);
15081
15082  out:
15083    WC_FREE_VAR_EX(aes, NULL, DYNAMIC_TYPE_AES);
15084
15085    return ret;
15086}
15087
15088int wc_AesKeyUnWrap_ex(Aes *aes, const byte* in, word32 inSz, byte* out,
15089        word32 outSz, const byte* iv)
15090{
15091    byte* r;
15092    word32 i, n;
15093    int j;
15094    int ret = 0;
15095
15096    byte t[KEYWRAP_BLOCK_SIZE];
15097    byte tmp[WC_AES_BLOCK_SIZE];
15098
15099    const byte* expIv;
15100    const byte defaultIV[] = {
15101        0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6
15102    };
15103
15104    if (aes == NULL || in == NULL || inSz < 3 * KEYWRAP_BLOCK_SIZE ||
15105        out == NULL || outSz < (inSz - KEYWRAP_BLOCK_SIZE))
15106        return BAD_FUNC_ARG;
15107
15108    /* input must be multiple of 64-bits */
15109    if (inSz % KEYWRAP_BLOCK_SIZE != 0)
15110        return BAD_FUNC_ARG;
15111
15112    /* user IV optional */
15113    if (iv != NULL)
15114        expIv = iv;
15115    else
15116        expIv = defaultIV;
15117
15118    /* A = C[0], R[i] = C[i] */
15119    XMEMCPY(tmp, in, KEYWRAP_BLOCK_SIZE);
15120    XMEMCPY(out, in + KEYWRAP_BLOCK_SIZE, inSz - KEYWRAP_BLOCK_SIZE);
15121    XMEMSET(t, 0, sizeof(t));
15122
15123    VECTOR_REGISTERS_PUSH;
15124
15125    /* initialize counter to 6n */
15126    n = (inSz - 1) / KEYWRAP_BLOCK_SIZE;
15127    InitKeyWrapCounter(t, 6 * n);
15128
15129    for (j = 5; j >= 0; j--) {
15130        for (i = n; i >= 1; i--) {
15131
15132            /* calculate A */
15133            xorbuf(tmp, t, KEYWRAP_BLOCK_SIZE);
15134            DecrementKeyWrapCounter(t);
15135
15136            /* load R[i], starting at end of R */
15137            r = out + ((i - 1) * KEYWRAP_BLOCK_SIZE);
15138            XMEMCPY(tmp + KEYWRAP_BLOCK_SIZE, r, KEYWRAP_BLOCK_SIZE);
15139            ret = wc_AesDecryptDirect(aes, tmp, tmp);
15140            if (ret != 0)
15141                break;
15142
15143            /* save R[i] */
15144            XMEMCPY(r, tmp + KEYWRAP_BLOCK_SIZE, KEYWRAP_BLOCK_SIZE);
15145        }
15146        if (ret != 0)
15147            break;
15148    }
15149
15150    VECTOR_REGISTERS_POP;
15151
15152    if (ret != 0)
15153        return ret;
15154
15155    /* verify IV */
15156    if (ConstantCompare(tmp, expIv, KEYWRAP_BLOCK_SIZE) != 0)
15157        return BAD_KEYWRAP_IV_E;
15158
15159    return (int)(inSz - KEYWRAP_BLOCK_SIZE);
15160}
15161
15162int wc_AesKeyUnWrap(const byte* key, word32 keySz, const byte* in, word32 inSz,
15163                    byte* out, word32 outSz, const byte* iv)
15164{
15165    WC_DECLARE_VAR(aes, Aes, 1, 0);
15166    int ret;
15167
15168    (void)iv;
15169
15170    if (key == NULL)
15171        return BAD_FUNC_ARG;
15172
15173#ifdef WOLFSSL_SMALL_STACK
15174    if ((aes = (Aes *)XMALLOC(sizeof *aes, NULL,
15175                              DYNAMIC_TYPE_AES)) == NULL)
15176        return MEMORY_E;
15177#endif
15178
15179
15180    ret = wc_AesInit(aes, NULL, INVALID_DEVID);
15181    if (ret != 0)
15182        goto out;
15183
15184    ret = wc_AesSetKey(aes, key, keySz, NULL, AES_DECRYPTION);
15185    if (ret != 0) {
15186        wc_AesFree(aes);
15187        goto out;
15188    }
15189
15190    ret = wc_AesKeyUnWrap_ex(aes, in, inSz, out, outSz, iv);
15191
15192    wc_AesFree(aes);
15193
15194  out:
15195    WC_FREE_VAR_EX(aes, NULL, DYNAMIC_TYPE_AES);
15196
15197    return ret;
15198}
15199
15200#endif /* HAVE_AES_KEYWRAP */
15201
15202#ifdef WOLFSSL_AES_XTS
15203
15204/* Galois Field to use */
15205#define GF_XTS 0x87
15206
15207/* Set up keys for encryption and/or decryption.
15208 *
15209 * aes   buffer holding aes subkeys
15210 * heap  heap hint to use for memory. Can be NULL
15211 * devId id to use with async crypto. Can be 0
15212 *
15213 * return 0 on success
15214 */
15215int wc_AesXtsInit(XtsAes* aes, void* heap, int devId)
15216{
15217    int    ret = 0;
15218
15219    if (aes == NULL) {
15220        return BAD_FUNC_ARG;
15221    }
15222
15223    if ((ret = wc_AesInit(&aes->tweak, heap, devId)) != 0) {
15224        return ret;
15225    }
15226    if ((ret = wc_AesInit(&aes->aes, heap, devId)) != 0) {
15227        (void)wc_AesFree(&aes->tweak);
15228        return ret;
15229    }
15230#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
15231    if ((ret = wc_AesInit(&aes->aes_decrypt, heap, devId)) != 0) {
15232        (void)wc_AesFree(&aes->tweak);
15233        (void)wc_AesFree(&aes->aes);
15234        return ret;
15235    }
15236#endif
15237
15238    return 0;
15239}
15240
15241/* Set up keys for encryption and/or decryption.
15242 *
15243 * aes   buffer holding aes subkeys
15244 * key   AES key for encrypt/decrypt and tweak process (concatenated)
15245 * len   length of key buffer in bytes. Should be twice that of key size. i.e.
15246 *       32 for a 16 byte key.
15247 * dir   direction: AES_ENCRYPTION, AES_DECRYPTION, or
15248 *       AES_ENCRYPTION_AND_DECRYPTION
15249 *
15250 * return 0 on success
15251 */
15252int wc_AesXtsSetKeyNoInit(XtsAes* aes, const byte* key, word32 len, int dir)
15253{
15254    word32 keySz;
15255    int    ret = 0;
15256
15257    if (aes == NULL || key == NULL) {
15258        return BAD_FUNC_ARG;
15259    }
15260
15261    if ((dir != AES_ENCRYPTION) && (dir != AES_DECRYPTION)
15262#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
15263        && (dir != AES_ENCRYPTION_AND_DECRYPTION)
15264#endif
15265        )
15266    {
15267        return BAD_FUNC_ARG;
15268    }
15269
15270    if ((len != (AES_128_KEY_SIZE*2)) &&
15271#ifndef HAVE_FIPS
15272        /* XTS-384 not allowed by FIPS and can not be treated like
15273         * RSA-4096 bit keys back in the day, can not vendor affirm
15274         * the use of 2 concatenated 192-bit keys (XTS-384) */
15275        (len != (AES_192_KEY_SIZE*2)) &&
15276#endif
15277        (len != (AES_256_KEY_SIZE*2)))
15278    {
15279        WOLFSSL_MSG("Unsupported key size");
15280        return WC_KEY_SIZE_E;
15281    }
15282
15283    keySz = len/2;
15284
15285#ifdef HAVE_FIPS
15286    if (XMEMCMP(key, key + keySz, keySz) == 0) {
15287        WOLFSSL_MSG("FIPS AES-XTS main and tweak keys must differ");
15288        return BAD_FUNC_ARG;
15289    }
15290#endif
15291
15292    if (dir == AES_ENCRYPTION
15293#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
15294        || dir == AES_ENCRYPTION_AND_DECRYPTION
15295#endif
15296        )
15297    {
15298        ret = wc_AesSetKey(&aes->aes, key, keySz, NULL, AES_ENCRYPTION);
15299    }
15300
15301#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
15302    if ((ret == 0) && ((dir == AES_DECRYPTION)
15303                       || (dir == AES_ENCRYPTION_AND_DECRYPTION)))
15304        ret = wc_AesSetKey(&aes->aes_decrypt, key, keySz, NULL, AES_DECRYPTION);
15305#else
15306    if (dir == AES_DECRYPTION)
15307        ret = wc_AesSetKey(&aes->aes, key, keySz, NULL, AES_DECRYPTION);
15308#endif
15309
15310    if (ret == 0)
15311        ret = wc_AesSetKey(&aes->tweak, key + keySz, keySz, NULL,
15312                AES_ENCRYPTION);
15313
15314#ifdef WOLFSSL_AESNI
15315    if (ret == 0) {
15316        /* With WC_C_DYNAMIC_FALLBACK, the main and tweak keys could have
15317         * conflicting _aesni status, but the AES-XTS asm implementations need
15318         * them to all be AESNI.  If any aren't, disable AESNI on all.
15319         */
15320    #ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
15321        if ((((dir == AES_ENCRYPTION) ||
15322              (dir == AES_ENCRYPTION_AND_DECRYPTION))
15323             && (aes->aes.use_aesni != aes->tweak.use_aesni))
15324            ||
15325            (((dir == AES_DECRYPTION) ||
15326              (dir == AES_ENCRYPTION_AND_DECRYPTION))
15327             && (aes->aes_decrypt.use_aesni != aes->tweak.use_aesni)))
15328        {
15329        #ifdef WC_C_DYNAMIC_FALLBACK
15330            aes->aes.use_aesni = 0;
15331            aes->aes_decrypt.use_aesni = 0;
15332            aes->tweak.use_aesni = 0;
15333        #else
15334            ret = SYSLIB_FAILED_E;
15335        #endif
15336        }
15337    #else /* !WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS */
15338        if (aes->aes.use_aesni != aes->tweak.use_aesni) {
15339        #ifdef WC_C_DYNAMIC_FALLBACK
15340            aes->aes.use_aesni = 0;
15341            aes->tweak.use_aesni = 0;
15342        #else
15343            ret = SYSLIB_FAILED_E;
15344        #endif
15345        }
15346    #endif /* !WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS */
15347    }
15348#endif /* WOLFSSL_AESNI */
15349
15350    return ret;
15351}
15352
15353/* Combined call to wc_AesXtsInit() and wc_AesXtsSetKeyNoInit().
15354 *
15355 * Note: is up to user to call wc_AesXtsFree when done.
15356 *
15357 * return 0 on success
15358 */
15359int wc_AesXtsSetKey(XtsAes* aes, const byte* key, word32 len, int dir,
15360        void* heap, int devId)
15361{
15362    int    ret = 0;
15363
15364    if (aes == NULL || key == NULL) {
15365        return BAD_FUNC_ARG;
15366    }
15367
15368    ret = wc_AesXtsInit(aes, heap, devId);
15369    if (ret != 0)
15370        return ret;
15371
15372    ret = wc_AesXtsSetKeyNoInit(aes, key, len, dir);
15373
15374    if (ret != 0)
15375        wc_AesXtsFree(aes);
15376
15377    return ret;
15378}
15379
15380
15381/* This is used to free up resources used by Aes structs
15382 *
15383 * aes AES keys to free
15384 *
15385 * return 0 on success
15386 */
15387int wc_AesXtsFree(XtsAes* aes)
15388{
15389    if (aes != NULL) {
15390        wc_AesFree(&aes->aes);
15391#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
15392        wc_AesFree(&aes->aes_decrypt);
15393#endif
15394        wc_AesFree(&aes->tweak);
15395    }
15396
15397    return 0;
15398}
15399
15400
15401/* Same process as wc_AesXtsEncrypt but uses a word64 type as the tweak value
15402 * instead of a byte array. This just converts the word64 to a byte array and
15403 * calls wc_AesXtsEncrypt.
15404 *
15405 * aes    AES keys to use for block encrypt/decrypt
15406 * out    output buffer to hold cipher text
15407 * in     input plain text buffer to encrypt
15408 * sz     size of both out and in buffers
15409 * sector value to use for tweak
15410 *
15411 * returns 0 on success
15412 */
15413int wc_AesXtsEncryptSector(XtsAes* aes, byte* out, const byte* in,
15414        word32 sz, word64 sector)
15415{
15416    byte* pt;
15417    byte  i[WC_AES_BLOCK_SIZE];
15418
15419    XMEMSET(i, 0, WC_AES_BLOCK_SIZE);
15420#ifdef BIG_ENDIAN_ORDER
15421    sector = ByteReverseWord64(sector);
15422#endif
15423    pt = (byte*)&sector;
15424    XMEMCPY(i, pt, sizeof(word64));
15425
15426    return wc_AesXtsEncrypt(aes, out, in, sz, (const byte*)i, WC_AES_BLOCK_SIZE);
15427}
15428
15429#ifdef HAVE_AES_DECRYPT
15430/* Same process as wc_AesXtsDecrypt but uses a word64 type as the tweak value
15431 * instead of a byte array. This just converts the word64 to a byte array.
15432 *
15433 * aes    AES keys to use for block encrypt/decrypt
15434 * out    output buffer to hold plain text
15435 * in     input cipher text buffer to encrypt
15436 * sz     size of both out and in buffers
15437 * sector value to use for tweak
15438 *
15439 * returns 0 on success
15440 */
15441int wc_AesXtsDecryptSector(XtsAes* aes, byte* out, const byte* in, word32 sz,
15442        word64 sector)
15443{
15444    byte* pt;
15445    byte  i[WC_AES_BLOCK_SIZE];
15446
15447    XMEMSET(i, 0, WC_AES_BLOCK_SIZE);
15448#ifdef BIG_ENDIAN_ORDER
15449    sector = ByteReverseWord64(sector);
15450#endif
15451    pt = (byte*)&sector;
15452    XMEMCPY(i, pt, sizeof(word64));
15453
15454    return wc_AesXtsDecrypt(aes, out, in, sz, (const byte*)i, WC_AES_BLOCK_SIZE);
15455}
15456#endif
15457
15458#ifdef WOLFSSL_AESNI
15459
15460#if defined(USE_INTEL_SPEEDUP_FOR_AES) && !defined(USE_INTEL_SPEEDUP)
15461    #define USE_INTEL_SPEEDUP
15462#endif
15463
15464#if defined(USE_INTEL_SPEEDUP)
15465    #define HAVE_INTEL_AVX1
15466    #define HAVE_INTEL_AVX2
15467#endif /* USE_INTEL_SPEEDUP */
15468
15469void AES_XTS_encrypt_aesni(const unsigned char *in, unsigned char *out, word32 sz,
15470                     const unsigned char* i, const unsigned char* key,
15471                     const unsigned char* key2, int nr)
15472                     XASM_LINK("AES_XTS_encrypt_aesni");
15473#ifdef WOLFSSL_AESXTS_STREAM
15474void AES_XTS_init_aesni(unsigned char* i, const unsigned char* tweak_key,
15475                     int tweak_nr)
15476                     XASM_LINK("AES_XTS_init_aesni");
15477void AES_XTS_encrypt_update_aesni(const unsigned char *in, unsigned char *out, word32 sz,
15478                     const unsigned char* key, unsigned char *i, int nr)
15479                     XASM_LINK("AES_XTS_encrypt_update_aesni");
15480#endif
15481#ifdef HAVE_INTEL_AVX1
15482void AES_XTS_encrypt_avx1(const unsigned char *in, unsigned char *out,
15483                     word32 sz, const unsigned char* i,
15484                     const unsigned char* key, const unsigned char* key2,
15485                     int nr)
15486                     XASM_LINK("AES_XTS_encrypt_avx1");
15487#ifdef WOLFSSL_AESXTS_STREAM
15488void AES_XTS_init_avx1(unsigned char* i, const unsigned char* tweak_key,
15489                     int tweak_nr)
15490                     XASM_LINK("AES_XTS_init_avx1");
15491void AES_XTS_encrypt_update_avx1(const unsigned char *in, unsigned char *out, word32 sz,
15492                     const unsigned char* key, unsigned char *i, int nr)
15493                     XASM_LINK("AES_XTS_encrypt_update_avx1");
15494#endif
15495#endif /* HAVE_INTEL_AVX1 */
15496
15497#ifdef HAVE_AES_DECRYPT
15498void AES_XTS_decrypt_aesni(const unsigned char *in, unsigned char *out, word32 sz,
15499                     const unsigned char* i, const unsigned char* key,
15500                     const unsigned char* key2, int nr)
15501                     XASM_LINK("AES_XTS_decrypt_aesni");
15502#ifdef WOLFSSL_AESXTS_STREAM
15503void AES_XTS_decrypt_update_aesni(const unsigned char *in, unsigned char *out, word32 sz,
15504                     const unsigned char* key, unsigned char *i, int nr)
15505                     XASM_LINK("AES_XTS_decrypt_update_aesni");
15506#endif
15507#ifdef HAVE_INTEL_AVX1
15508void AES_XTS_decrypt_avx1(const unsigned char *in, unsigned char *out,
15509                     word32 sz, const unsigned char* i,
15510                     const unsigned char* key, const unsigned char* key2,
15511                     int nr)
15512                     XASM_LINK("AES_XTS_decrypt_avx1");
15513#ifdef WOLFSSL_AESXTS_STREAM
15514void AES_XTS_decrypt_update_avx1(const unsigned char *in, unsigned char *out, word32 sz,
15515                     const unsigned char* key, unsigned char *i, int nr)
15516                     XASM_LINK("AES_XTS_decrypt_update_avx1");
15517#endif
15518#endif /* HAVE_INTEL_AVX1 */
15519#endif /* HAVE_AES_DECRYPT */
15520
15521#endif /* WOLFSSL_AESNI */
15522
15523#ifdef HAVE_AES_ECB
15524#if (!defined(WOLFSSL_ARMASM) || (!defined(__aarch64__) && \
15525    defined(WOLFSSL_ARMASM_NO_HW_CRYPTO))) || defined(WOLFSSL_AESXTS_STREAM)
15526/* helper function for encrypting / decrypting full buffer at once */
15527static WARN_UNUSED_RESULT int _AesXtsHelper(
15528    Aes* aes, byte* out, const byte* in, word32 sz, int dir)
15529{
15530    word32 outSz   = sz;
15531    word32 totalSz = (sz / WC_AES_BLOCK_SIZE) * WC_AES_BLOCK_SIZE; /* total bytes */
15532    byte*  pt      = out;
15533
15534    outSz -= WC_AES_BLOCK_SIZE;
15535
15536    while (outSz > 0) {
15537        word32 j;
15538        byte carry = 0;
15539
15540        /* multiply by shift left and propagate carry */
15541        for (j = 0; j < WC_AES_BLOCK_SIZE && outSz > 0; j++, outSz--) {
15542            byte tmpC;
15543
15544            tmpC   = (pt[j] >> 7) & 0x01;
15545            pt[j+WC_AES_BLOCK_SIZE] = (byte)((pt[j] << 1) + carry);
15546            carry  = tmpC;
15547        }
15548        if (carry) {
15549            pt[WC_AES_BLOCK_SIZE] ^= GF_XTS;
15550        }
15551
15552        pt += WC_AES_BLOCK_SIZE;
15553    }
15554
15555    xorbuf(out, in, totalSz);
15556#ifndef WOLFSSL_RISCV_ASM
15557    if (dir == AES_ENCRYPTION) {
15558        return _AesEcbEncrypt(aes, out, out, totalSz);
15559    }
15560    else {
15561        return _AesEcbDecrypt(aes, out, out, totalSz);
15562    }
15563#else
15564    if (dir == AES_ENCRYPTION) {
15565        return wc_AesEcbEncrypt(aes, out, out, totalSz);
15566    }
15567    else {
15568        return wc_AesEcbDecrypt(aes, out, out, totalSz);
15569    }
15570#endif
15571}
15572#endif
15573#endif /* HAVE_AES_ECB */
15574
15575/* AES with XTS mode. (XTS) XEX encryption with Tweak and cipher text Stealing.
15576 *
15577 * xaes  AES keys to use for block encrypt/decrypt
15578 * out   output buffer to hold cipher text
15579 * in    input plain text buffer to encrypt
15580 * sz    size of both out and in buffers
15581 * i     value to use for tweak
15582 *
15583 * returns 0 on success
15584 */
15585/* Software AES - XTS Encrypt  */
15586
15587#if !defined(WOLFSSL_ARMASM) || (!defined(__aarch64__) && \
15588    defined(WOLFSSL_ARMASM_NO_HW_CRYPTO))
15589static int AesXtsEncryptUpdate_sw(XtsAes* xaes, byte* out, const byte* in,
15590                                  word32 sz,
15591                                  byte *i);
15592static int AesXtsEncrypt_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz,
15593        const byte* i)
15594{
15595    int ret;
15596    byte tweak_block[WC_AES_BLOCK_SIZE];
15597
15598    ret = wc_AesEncryptDirect(&xaes->tweak, tweak_block, i);
15599    if (ret != 0)
15600        return ret;
15601
15602    return AesXtsEncryptUpdate_sw(xaes, out, in, sz, tweak_block);
15603}
15604#endif
15605
15606#ifdef WOLFSSL_AESXTS_STREAM
15607
15608/* Block-streaming AES-XTS tweak setup.
15609 *
15610 * xaes  AES keys to use for block encrypt/decrypt
15611 * i     readwrite value to use for tweak
15612 *
15613 * returns 0 on success
15614 */
15615static int AesXtsInitTweak_sw(XtsAes* xaes, byte* i) {
15616    return wc_AesEncryptDirect(&xaes->tweak, i, i);
15617}
15618
15619#endif /* WOLFSSL_AESXTS_STREAM */
15620
15621#if !defined(WOLFSSL_ARMASM) || (!defined(__aarch64__) && \
15622    defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)) || defined(WOLFSSL_AESXTS_STREAM)
15623/* Block-streaming AES-XTS.
15624 *
15625 * Supply block-aligned input data with successive calls.  Final call need not
15626 * be block aligned.
15627 *
15628 * xaes  AES keys to use for block encrypt/decrypt
15629 * out   output buffer to hold cipher text
15630 * in    input plain text buffer to encrypt
15631 * sz    size of both out and in buffers
15632 *
15633 * returns 0 on success
15634 */
15635/* Software AES - XTS Encrypt  */
15636static int AesXtsEncryptUpdate_sw(XtsAes* xaes, byte* out, const byte* in,
15637                                  word32 sz,
15638                                  byte *i)
15639{
15640    int ret = 0;
15641    word32 blocks = (sz / WC_AES_BLOCK_SIZE);
15642    Aes *aes = &xaes->aes;
15643
15644#ifdef HAVE_AES_ECB
15645    /* encrypt all of buffer at once when possible */
15646    if (in != out) { /* can not handle inline */
15647        XMEMCPY(out, i, WC_AES_BLOCK_SIZE);
15648        if ((ret = _AesXtsHelper(aes, out, in, sz, AES_ENCRYPTION)) != 0)
15649            return ret;
15650    }
15651#endif
15652
15653    while (blocks > 0) {
15654        word32 j;
15655        byte carry = 0;
15656
15657#ifdef HAVE_AES_ECB
15658        if (in == out)
15659#endif
15660        { /* check for if inline */
15661            byte buf[WC_AES_BLOCK_SIZE];
15662
15663            XMEMCPY(buf, in, WC_AES_BLOCK_SIZE);
15664            xorbuf(buf, i, WC_AES_BLOCK_SIZE);
15665            ret = wc_AesEncryptDirect(aes, out, buf);
15666            if (ret != 0)
15667                return ret;
15668        }
15669        xorbuf(out, i, WC_AES_BLOCK_SIZE);
15670
15671        /* multiply by shift left and propagate carry */
15672        for (j = 0; j < WC_AES_BLOCK_SIZE; j++) {
15673            byte tmpC;
15674
15675            tmpC   = (i[j] >> 7) & 0x01;
15676            i[j] = (byte)((i[j] << 1) + carry);
15677            carry  = tmpC;
15678        }
15679        if (carry) {
15680            i[0] ^= GF_XTS;
15681        }
15682
15683        in  += WC_AES_BLOCK_SIZE;
15684        out += WC_AES_BLOCK_SIZE;
15685        sz  -= WC_AES_BLOCK_SIZE;
15686        blocks--;
15687    }
15688
15689    /* stealing operation of XTS to handle left overs */
15690    if (sz > 0) {
15691        byte buf[WC_AES_BLOCK_SIZE];
15692
15693        XMEMCPY(buf, out - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
15694        if (sz >= WC_AES_BLOCK_SIZE) { /* extra sanity check before copy */
15695            return BUFFER_E;
15696        }
15697        if (in != out) {
15698            XMEMCPY(out, buf, sz);
15699            XMEMCPY(buf, in, sz);
15700        }
15701        else {
15702            byte buf2[WC_AES_BLOCK_SIZE];
15703
15704            XMEMCPY(buf2, buf, sz);
15705            XMEMCPY(buf, in, sz);
15706            XMEMCPY(out, buf2, sz);
15707        }
15708
15709        xorbuf(buf, i, WC_AES_BLOCK_SIZE);
15710        ret = wc_AesEncryptDirect(aes, out - WC_AES_BLOCK_SIZE, buf);
15711        if (ret == 0)
15712            xorbuf(out - WC_AES_BLOCK_SIZE, i, WC_AES_BLOCK_SIZE);
15713    }
15714
15715    return ret;
15716}
15717#endif
15718
15719/* AES with XTS mode. (XTS) XEX encryption with Tweak and cipher text Stealing.
15720 *
15721 * xaes  AES keys to use for block encrypt/decrypt
15722 * out   output buffer to hold cipher text
15723 * in    input plain text buffer to encrypt
15724 * sz    size of both out and in buffers
15725 * i     value to use for tweak
15726 * iSz   size of i buffer, should always be WC_AES_BLOCK_SIZE but having this input
15727 *       adds a sanity check on how the user calls the function.
15728 *
15729 * returns 0 on success
15730 */
15731int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
15732        const byte* i, word32 iSz)
15733{
15734    int ret;
15735
15736    Aes *aes;
15737
15738    if (xaes == NULL || out == NULL || in == NULL) {
15739        return BAD_FUNC_ARG;
15740    }
15741
15742#if FIPS_VERSION3_GE(6,0,0)
15743    /* SP800-38E - Restrict data unit to 2^20 blocks per key. A block is
15744     * WC_AES_BLOCK_SIZE or 16-bytes (128-bits). So each key may only be used to
15745     * protect up to 1,048,576 blocks of WC_AES_BLOCK_SIZE (16,777,216 bytes)
15746     */
15747    if (sz > FIPS_AES_XTS_MAX_BYTES_PER_TWEAK) {
15748        WOLFSSL_MSG("Request exceeds allowed bytes per SP800-38E");
15749        return BAD_FUNC_ARG;
15750    }
15751#endif
15752
15753    aes = &xaes->aes;
15754
15755    if (aes->keylen == 0) {
15756        WOLFSSL_MSG("wc_AesXtsEncrypt called with unset encryption key.");
15757        return BAD_FUNC_ARG;
15758    }
15759
15760    if (iSz < WC_AES_BLOCK_SIZE) {
15761        return BAD_FUNC_ARG;
15762    }
15763
15764    if (sz < WC_AES_BLOCK_SIZE) {
15765        WOLFSSL_MSG("Plain text input too small for encryption");
15766        return BAD_FUNC_ARG;
15767    }
15768
15769#if !defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
15770      !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
15771    AES_XTS_encrypt_AARCH32(in, out, sz, i, (byte*)xaes->aes.key,
15772        (byte*)xaes->tweak.key, (byte*)xaes->aes.tmp, xaes->aes.rounds);
15773    ret = 0;
15774#elif defined(WOLFSSL_AESNI)
15775    if (aes->use_aesni) {
15776        SAVE_VECTOR_REGISTERS(return _svr_ret;);
15777#if defined(HAVE_INTEL_AVX1)
15778        if (IS_INTEL_AVX1(intel_flags)) {
15779            AES_XTS_encrypt_avx1(in, out, sz, i,
15780                                 (const byte*)aes->key,
15781                                 (const byte*)xaes->tweak.key,
15782                                 (int)aes->rounds);
15783            ret = 0;
15784        }
15785        else
15786#endif
15787        {
15788            AES_XTS_encrypt_aesni(in, out, sz, i,
15789                                  (const byte*)aes->key,
15790                                  (const byte*)xaes->tweak.key,
15791                                  (int)aes->rounds);
15792            ret = 0;
15793        }
15794        RESTORE_VECTOR_REGISTERS();
15795    }
15796    else {
15797        ret = AesXtsEncrypt_sw(xaes, out, in, sz, i);
15798    }
15799#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM)
15800#if !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
15801    if (aes->use_aes_hw_crypto) {
15802        AES_XTS_encrypt_AARCH64(in, out, sz, i, (byte*)xaes->aes.key,
15803            (byte*)xaes->tweak.key, (byte*)xaes->aes.tmp, xaes->aes.rounds);
15804        ret = 0;
15805    }
15806    else
15807#endif
15808#if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON)
15809#ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
15810    if (sz >= 32)
15811#endif
15812    {
15813        AES_XTS_encrypt_NEON(in, out, sz, i, (byte*)xaes->aes.key,
15814            (byte*)xaes->tweak.key, (byte*)xaes->aes.tmp, xaes->aes.rounds);
15815        ret = 0;
15816    }
15817#ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
15818    else
15819#endif
15820#endif
15821#ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
15822    {
15823        AES_XTS_encrypt(in, out, sz, i, (byte*)xaes->aes.key,
15824            (byte*)xaes->tweak.key, (byte*)xaes->aes.tmp, xaes->aes.rounds);
15825        ret = 0;
15826    }
15827#endif
15828#else
15829    ret = AesXtsEncrypt_sw(xaes, out, in, sz, i);
15830#endif
15831
15832    return ret;
15833}
15834
15835#ifdef WOLFSSL_AESXTS_STREAM
15836
15837/* Block-streaming AES-XTS.
15838 *
15839 * xaes  AES keys to use for block encrypt/decrypt
15840 * i     readwrite value to use for tweak
15841 * iSz   size of i buffer, should always be WC_AES_BLOCK_SIZE but having this input
15842 *       adds a sanity check on how the user calls the function.
15843 *
15844 * returns 0 on success
15845 */
15846int wc_AesXtsEncryptInit(XtsAes* xaes, const byte* i, word32 iSz,
15847                         struct XtsAesStreamData *stream)
15848{
15849    int ret;
15850
15851    Aes *aes;
15852
15853    if ((xaes == NULL) || (i == NULL) || (stream == NULL)) {
15854        return BAD_FUNC_ARG;
15855    }
15856
15857    if (iSz < WC_AES_BLOCK_SIZE) {
15858        return BAD_FUNC_ARG;
15859    }
15860
15861    aes = &xaes->aes;
15862
15863    if (aes->keylen == 0) {
15864        WOLFSSL_MSG("wc_AesXtsEncrypt called with unset encryption key.");
15865        return BAD_FUNC_ARG;
15866    }
15867
15868    XMEMCPY(stream->tweak_block, i, WC_AES_BLOCK_SIZE);
15869    stream->bytes_crypted_with_this_tweak = 0;
15870
15871    {
15872#ifdef WOLFSSL_AESNI
15873        if (aes->use_aesni) {
15874            SAVE_VECTOR_REGISTERS(return _svr_ret;);
15875#if defined(HAVE_INTEL_AVX1)
15876            if (IS_INTEL_AVX1(intel_flags)) {
15877                AES_XTS_init_avx1(stream->tweak_block,
15878                                  (const byte*)xaes->tweak.key,
15879                                  (int)xaes->tweak.rounds);
15880                ret = 0;
15881            }
15882            else
15883#endif
15884            {
15885                AES_XTS_init_aesni(stream->tweak_block,
15886                                   (const byte*)xaes->tweak.key,
15887                                   (int)xaes->tweak.rounds);
15888                ret = 0;
15889            }
15890            RESTORE_VECTOR_REGISTERS();
15891        }
15892        else
15893#endif /* WOLFSSL_AESNI */
15894        {
15895            ret = AesXtsInitTweak_sw(xaes, stream->tweak_block);
15896        }
15897    }
15898
15899    return ret;
15900}
15901
15902/* Block-streaming AES-XTS
15903 *
15904 * Note that sz must be >= WC_AES_BLOCK_SIZE in each call, and must be a multiple
15905 * of WC_AES_BLOCK_SIZE in each call to wc_AesXtsEncryptUpdate().
15906 * wc_AesXtsEncryptFinal() can handle any length >= WC_AES_BLOCK_SIZE.
15907 *
15908 * xaes  AES keys to use for block encrypt/decrypt
15909 * out   output buffer to hold cipher text
15910 * in    input plain text buffer to encrypt
15911 * sz    size of both out and in buffers -- must be >= WC_AES_BLOCK_SIZE.
15912 * i     value to use for tweak
15913 * iSz   size of i buffer, should always be WC_AES_BLOCK_SIZE but having this input
15914 *       adds a sanity check on how the user calls the function.
15915 *
15916 * returns 0 on success
15917 */
15918static int AesXtsEncryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 sz,
15919                           struct XtsAesStreamData *stream)
15920{
15921    int ret;
15922
15923#ifdef WOLFSSL_AESNI
15924    Aes *aes;
15925#endif
15926
15927    if (xaes == NULL || out == NULL || in == NULL) {
15928        return BAD_FUNC_ARG;
15929    }
15930
15931#ifdef WOLFSSL_AESNI
15932    aes = &xaes->aes;
15933#endif
15934
15935    if (sz < WC_AES_BLOCK_SIZE) {
15936        WOLFSSL_MSG("Plain text input too small for encryption");
15937        return BAD_FUNC_ARG;
15938    }
15939
15940    if (stream->bytes_crypted_with_this_tweak & ((word32)WC_AES_BLOCK_SIZE - 1U))
15941    {
15942        WOLFSSL_MSG("Call to AesXtsEncryptUpdate after previous finalizing call");
15943        return BAD_FUNC_ARG;
15944    }
15945
15946#ifndef WC_AESXTS_STREAM_NO_REQUEST_ACCOUNTING
15947    if (! WC_SAFE_SUM_WORD32(stream->bytes_crypted_with_this_tweak, sz,
15948                             stream->bytes_crypted_with_this_tweak))
15949    {
15950        WOLFSSL_MSG("Overflow of stream->bytes_crypted_with_this_tweak "
15951                    "in AesXtsEncryptUpdate().");
15952    }
15953#endif
15954#if FIPS_VERSION3_GE(6,0,0)
15955    /* SP800-38E - Restrict data unit to 2^20 blocks per key. A block is
15956     * WC_AES_BLOCK_SIZE or 16-bytes (128-bits). So each key may only be used to
15957     * protect up to 1,048,576 blocks of WC_AES_BLOCK_SIZE (16,777,216 bytes)
15958     */
15959    if (stream->bytes_crypted_with_this_tweak >
15960        FIPS_AES_XTS_MAX_BYTES_PER_TWEAK)
15961    {
15962        WOLFSSL_MSG("Request exceeds allowed bytes per SP800-38E");
15963        return BAD_FUNC_ARG;
15964    }
15965#endif
15966    {
15967#ifdef WOLFSSL_AESNI
15968        if (aes->use_aesni) {
15969            SAVE_VECTOR_REGISTERS(return _svr_ret;);
15970#if defined(HAVE_INTEL_AVX1)
15971            if (IS_INTEL_AVX1(intel_flags)) {
15972                AES_XTS_encrypt_update_avx1(in, out, sz,
15973                                            (const byte*)aes->key,
15974                                            stream->tweak_block,
15975                                            (int)aes->rounds);
15976                ret = 0;
15977            }
15978            else
15979#endif
15980            {
15981                AES_XTS_encrypt_update_aesni(in, out, sz,
15982                                            (const byte*)aes->key,
15983                                            stream->tweak_block,
15984                                            (int)aes->rounds);
15985                ret = 0;
15986            }
15987            RESTORE_VECTOR_REGISTERS();
15988        }
15989        else
15990#endif /* WOLFSSL_AESNI */
15991        {
15992            ret = AesXtsEncryptUpdate_sw(xaes, out, in, sz, stream->tweak_block);
15993        }
15994    }
15995
15996    return ret;
15997}
15998
15999int wc_AesXtsEncryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 sz,
16000                           struct XtsAesStreamData *stream)
16001{
16002    if (stream == NULL)
16003        return BAD_FUNC_ARG;
16004    if (sz & ((word32)WC_AES_BLOCK_SIZE - 1U))
16005        return BAD_FUNC_ARG;
16006    return AesXtsEncryptUpdate(xaes, out, in, sz, stream);
16007}
16008
16009int wc_AesXtsEncryptFinal(XtsAes* xaes, byte* out, const byte* in, word32 sz,
16010                           struct XtsAesStreamData *stream)
16011{
16012    int ret;
16013    if (stream == NULL)
16014        return BAD_FUNC_ARG;
16015    if (sz > 0)
16016        ret = AesXtsEncryptUpdate(xaes, out, in, sz, stream);
16017    else
16018        ret = 0;
16019    /* force the count odd, to assure error on attempt to AesXtsEncryptUpdate()
16020     * after finalization.
16021     */
16022    stream->bytes_crypted_with_this_tweak |= 1U;
16023    ForceZero(stream->tweak_block, WC_AES_BLOCK_SIZE);
16024#ifdef WOLFSSL_CHECK_MEM_ZERO
16025    wc_MemZero_Check(stream->tweak_block, WC_AES_BLOCK_SIZE);
16026#endif
16027    return ret;
16028}
16029
16030#endif /* WOLFSSL_AESXTS_STREAM */
16031
16032#ifdef HAVE_AES_DECRYPT
16033
16034/* Same process as encryption but use aes_decrypt key.
16035 *
16036 * xaes  AES keys to use for block encrypt/decrypt
16037 * out   output buffer to hold plain text
16038 * in    input cipher text buffer to decrypt
16039 * sz    size of both out and in buffers
16040 * i     value to use for tweak
16041 *
16042 * returns 0 on success
16043 */
16044/* Software AES - XTS Decrypt */
16045
16046#if !defined(WOLFSSL_ARMASM) || (!defined(__aarch64__) && \
16047    defined(WOLFSSL_ARMASM_NO_HW_CRYPTO))
16048static int AesXtsDecryptUpdate_sw(XtsAes* xaes, byte* out, const byte* in,
16049                                  word32 sz, byte *i);
16050
16051static int AesXtsDecrypt_sw(XtsAes* xaes, byte* out, const byte* in, word32 sz,
16052        const byte* i)
16053{
16054    int ret;
16055    byte tweak_block[WC_AES_BLOCK_SIZE];
16056
16057    ret = wc_AesEncryptDirect(&xaes->tweak, tweak_block, i);
16058    if (ret != 0)
16059        return ret;
16060
16061    return AesXtsDecryptUpdate_sw(xaes, out, in, sz, tweak_block);
16062}
16063#endif
16064
16065#if (!defined(WOLFSSL_ARMASM) || (!defined(__aarch64__) && \
16066    defined(WOLFSSL_ARMASM_NO_HW_CRYPTO))) || defined(WOLFSSL_AESXTS_STREAM)
16067/* Block-streaming AES-XTS.
16068 *
16069 * Same process as encryption but use decrypt key.
16070 *
16071 * Supply block-aligned input data with successive calls.  Final call need not
16072 * be block aligned.
16073 *
16074 * xaes  AES keys to use for block encrypt/decrypt
16075 * out   output buffer to hold plain text
16076 * in    input cipher text buffer to decrypt
16077 * sz    size of both out and in buffers
16078 * i     value to use for tweak
16079 *
16080 * returns 0 on success
16081 */
16082/* Software AES - XTS Decrypt */
16083static int AesXtsDecryptUpdate_sw(XtsAes* xaes, byte* out, const byte* in,
16084                                  word32 sz, byte *i)
16085{
16086    int ret = 0;
16087    word32 blocks = (sz / WC_AES_BLOCK_SIZE);
16088#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
16089    Aes *aes = &xaes->aes_decrypt;
16090#else
16091    Aes *aes = &xaes->aes;
16092#endif
16093    word32 j;
16094    byte carry = 0;
16095    byte stl = (sz % WC_AES_BLOCK_SIZE);
16096
16097    /* if Stealing then break out of loop one block early to handle special
16098     * case */
16099    if (stl > 0) {
16100        blocks--;
16101    }
16102
16103#ifdef HAVE_AES_ECB
16104    /* decrypt all of buffer at once when possible */
16105    if (in != out) { /* can not handle inline */
16106        XMEMCPY(out, i, WC_AES_BLOCK_SIZE);
16107        if ((ret = _AesXtsHelper(aes, out, in, sz, AES_DECRYPTION)) != 0)
16108            return ret;
16109    }
16110#endif
16111
16112    while (blocks > 0) {
16113#ifdef HAVE_AES_ECB
16114        if (in == out)
16115#endif
16116        { /* check for if inline */
16117            byte buf[WC_AES_BLOCK_SIZE];
16118
16119            XMEMCPY(buf, in, WC_AES_BLOCK_SIZE);
16120            xorbuf(buf, i, WC_AES_BLOCK_SIZE);
16121            ret = wc_AesDecryptDirect(aes, out, buf);
16122            if (ret != 0)
16123                return ret;
16124        }
16125        xorbuf(out, i, WC_AES_BLOCK_SIZE);
16126
16127        /* multiply by shift left and propagate carry */
16128        for (j = 0; j < WC_AES_BLOCK_SIZE; j++) {
16129            byte tmpC;
16130
16131            tmpC   = (i[j] >> 7) & 0x01;
16132            i[j] = (byte)((i[j] << 1) + carry);
16133            carry  = tmpC;
16134        }
16135        if (carry) {
16136            i[0] ^= GF_XTS;
16137        }
16138        carry = 0;
16139
16140        in  += WC_AES_BLOCK_SIZE;
16141        out += WC_AES_BLOCK_SIZE;
16142        sz  -= WC_AES_BLOCK_SIZE;
16143        blocks--;
16144    }
16145
16146    /* stealing operation of XTS to handle left overs */
16147    if (sz >= WC_AES_BLOCK_SIZE) {
16148        byte buf[WC_AES_BLOCK_SIZE];
16149        byte tmp2[WC_AES_BLOCK_SIZE];
16150
16151        /* multiply by shift left and propagate carry */
16152        for (j = 0; j < WC_AES_BLOCK_SIZE; j++) {
16153            byte tmpC;
16154
16155            tmpC   = (i[j] >> 7) & 0x01;
16156            tmp2[j] = (byte)((i[j] << 1) + carry);
16157            carry  = tmpC;
16158        }
16159        if (carry) {
16160            tmp2[0] ^= GF_XTS;
16161        }
16162
16163        XMEMCPY(buf, in, WC_AES_BLOCK_SIZE);
16164        xorbuf(buf, tmp2, WC_AES_BLOCK_SIZE);
16165        ret = wc_AesDecryptDirect(aes, out, buf);
16166        if (ret != 0)
16167            return ret;
16168        xorbuf(out, tmp2, WC_AES_BLOCK_SIZE);
16169
16170        /* tmp2 holds partial | last */
16171        XMEMCPY(tmp2, out, WC_AES_BLOCK_SIZE);
16172        in  += WC_AES_BLOCK_SIZE;
16173        out += WC_AES_BLOCK_SIZE;
16174        sz  -= WC_AES_BLOCK_SIZE;
16175
16176        /* Make buffer with end of cipher text | last */
16177        XMEMCPY(buf, tmp2, WC_AES_BLOCK_SIZE);
16178        if (sz >= WC_AES_BLOCK_SIZE) { /* extra sanity check before copy */
16179            return BUFFER_E;
16180        }
16181        XMEMCPY(buf, in,   sz);
16182        XMEMCPY(out, tmp2, sz);
16183
16184        xorbuf(buf, i, WC_AES_BLOCK_SIZE);
16185        ret = wc_AesDecryptDirect(aes, tmp2, buf);
16186        if (ret != 0)
16187            return ret;
16188        xorbuf(tmp2, i, WC_AES_BLOCK_SIZE);
16189        XMEMCPY(out - WC_AES_BLOCK_SIZE, tmp2, WC_AES_BLOCK_SIZE);
16190    }
16191
16192    return ret;
16193}
16194#endif
16195
16196/* Same process as encryption but Aes key is AES_DECRYPTION type.
16197 *
16198 * xaes  AES keys to use for block encrypt/decrypt
16199 * out   output buffer to hold plain text
16200 * in    input cipher text buffer to decrypt
16201 * sz    size of both out and in buffers
16202 * i     value to use for tweak
16203 * iSz   size of i buffer, should always be WC_AES_BLOCK_SIZE but having this input
16204 *       adds a sanity check on how the user calls the function.
16205 *
16206 * returns 0 on success
16207 */
16208int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
16209        const byte* i, word32 iSz)
16210{
16211    int ret;
16212    Aes *aes;
16213
16214    if (xaes == NULL || out == NULL || in == NULL) {
16215        return BAD_FUNC_ARG;
16216    }
16217
16218#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
16219    aes = &xaes->aes_decrypt;
16220#else
16221    aes = &xaes->aes;
16222#endif
16223
16224/* FIPS TODO: SP800-38E - Restrict data unit to 2^20 blocks per key. A block is
16225 * WC_AES_BLOCK_SIZE or 16-bytes (128-bits). So each key may only be used to
16226 * protect up to 1,048,576 blocks of WC_AES_BLOCK_SIZE (16,777,216 bytes or
16227 * 134,217,728-bits) Add helpful printout and message along with BAD_FUNC_ARG
16228 * return whenever sz / WC_AES_BLOCK_SIZE > 1,048,576 or equal to that and sz is
16229 * not a sequence of complete blocks.
16230 */
16231
16232    if (aes->keylen == 0) {
16233        WOLFSSL_MSG("wc_AesXtsDecrypt called with unset decryption key.");
16234        return BAD_FUNC_ARG;
16235    }
16236
16237    if (iSz < WC_AES_BLOCK_SIZE) {
16238        return BAD_FUNC_ARG;
16239    }
16240
16241    if (sz < WC_AES_BLOCK_SIZE) {
16242        WOLFSSL_MSG("Cipher text input too small for decryption");
16243        return BAD_FUNC_ARG;
16244    }
16245
16246#if !defined(__aarch64__) && defined(WOLFSSL_ARMASM) && \
16247      !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
16248    AES_XTS_decrypt_AARCH32(in, out, sz, i, (byte*)xaes->aes.key,
16249        (byte*)xaes->tweak.key, (byte*)xaes->aes.tmp, xaes->aes.rounds);
16250    ret = 0;
16251#elif defined(WOLFSSL_AESNI)
16252    if (aes->use_aesni) {
16253        SAVE_VECTOR_REGISTERS(return _svr_ret;);
16254#if defined(HAVE_INTEL_AVX1)
16255        if (IS_INTEL_AVX1(intel_flags)) {
16256            AES_XTS_decrypt_avx1(in, out, sz, i,
16257                                 (const byte*)aes->key,
16258                                 (const byte*)xaes->tweak.key,
16259                                 (int)aes->rounds);
16260            ret = 0;
16261        }
16262        else
16263#endif
16264        {
16265            AES_XTS_decrypt_aesni(in, out, sz, i,
16266                                  (const byte*)aes->key,
16267                                  (const byte*)xaes->tweak.key,
16268                                  (int)aes->rounds);
16269            ret = 0;
16270        }
16271        RESTORE_VECTOR_REGISTERS();
16272    }
16273    else {
16274        ret = AesXtsDecrypt_sw(xaes, out, in, sz, i);
16275    }
16276#elif defined(__aarch64__) && defined(WOLFSSL_ARMASM)
16277#if !defined(WOLFSSL_ARMASM_NO_HW_CRYPTO)
16278    if (aes->use_aes_hw_crypto) {
16279        AES_XTS_decrypt_AARCH64(in, out, sz, i, (byte*)xaes->aes.key,
16280            (byte*)xaes->tweak.key, (byte*)xaes->aes.tmp, xaes->aes.rounds);
16281        ret = 0;
16282    }
16283    else
16284#endif
16285#if defined(__aarch64__) && !defined(WOLFSSL_ARMASM_NO_NEON)
16286#ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
16287    if (sz >= 64)
16288#endif
16289    {
16290        AES_XTS_decrypt_NEON(in, out, sz, i, (byte*)xaes->aes.key,
16291            (byte*)xaes->tweak.key, (byte*)xaes->aes.tmp, xaes->aes.rounds);
16292        ret = 0;
16293    }
16294#ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
16295    else
16296#endif
16297#endif
16298#ifndef WOLFSSL_ARMASM_NEON_NO_TABLE_LOOKUP
16299    {
16300        AES_XTS_decrypt(in, out, sz, i, (byte*)xaes->aes.key,
16301            (byte*)xaes->tweak.key, (byte*)xaes->aes.tmp, xaes->aes.rounds);
16302        ret = 0;
16303    }
16304#endif
16305#else
16306    ret = AesXtsDecrypt_sw(xaes, out, in, sz, i);
16307#endif
16308
16309    return ret;
16310}
16311
16312#ifdef WOLFSSL_AESXTS_STREAM
16313
16314/* Same process as encryption but Aes key is AES_DECRYPTION type.
16315 *
16316 * xaes  AES keys to use for block encrypt/decrypt
16317 * i     readwrite value to use for tweak
16318 * iSz   size of i buffer, should always be WC_AES_BLOCK_SIZE but having this input
16319 *       adds a sanity check on how the user calls the function.
16320 *
16321 * returns 0 on success
16322 */
16323int wc_AesXtsDecryptInit(XtsAes* xaes, const byte* i, word32 iSz,
16324                         struct XtsAesStreamData *stream)
16325{
16326    int ret;
16327    Aes *aes;
16328
16329    if (xaes == NULL) {
16330        return BAD_FUNC_ARG;
16331    }
16332
16333#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
16334    aes = &xaes->aes_decrypt;
16335#else
16336    aes = &xaes->aes;
16337#endif
16338
16339    if (aes->keylen == 0) {
16340        WOLFSSL_MSG("wc_AesXtsDecrypt called with unset decryption key.");
16341        return BAD_FUNC_ARG;
16342    }
16343
16344    if (iSz < WC_AES_BLOCK_SIZE) {
16345        return BAD_FUNC_ARG;
16346    }
16347
16348    XMEMCPY(stream->tweak_block, i, WC_AES_BLOCK_SIZE);
16349    stream->bytes_crypted_with_this_tweak = 0;
16350
16351    {
16352#ifdef WOLFSSL_AESNI
16353        if (aes->use_aesni) {
16354            SAVE_VECTOR_REGISTERS(return _svr_ret;);
16355#if defined(HAVE_INTEL_AVX1)
16356            if (IS_INTEL_AVX1(intel_flags)) {
16357                AES_XTS_init_avx1(stream->tweak_block,
16358                                  (const byte*)xaes->tweak.key,
16359                                  (int)xaes->tweak.rounds);
16360                ret = 0;
16361            }
16362            else
16363#endif
16364            {
16365                AES_XTS_init_aesni(stream->tweak_block,
16366                                   (const byte*)xaes->tweak.key,
16367                                   (int)xaes->tweak.rounds);
16368                ret = 0;
16369            }
16370            RESTORE_VECTOR_REGISTERS();
16371        }
16372        else
16373#endif /* WOLFSSL_AESNI */
16374        {
16375            ret = AesXtsInitTweak_sw(xaes, stream->tweak_block);
16376        }
16377
16378    }
16379
16380    return ret;
16381}
16382
16383/* Block-streaming AES-XTS
16384 *
16385 * Note that sz must be >= WC_AES_BLOCK_SIZE in each call, and must be a multiple
16386 * of WC_AES_BLOCK_SIZE in each call to wc_AesXtsDecryptUpdate().
16387 * wc_AesXtsDecryptFinal() can handle any length >= WC_AES_BLOCK_SIZE.
16388 *
16389 * xaes  AES keys to use for block encrypt/decrypt
16390 * out   output buffer to hold plain text
16391 * in    input cipher text buffer to decrypt
16392 * sz    size of both out and in buffers
16393 * i     tweak buffer of size WC_AES_BLOCK_SIZE.
16394 *
16395 * returns 0 on success
16396 */
16397static int AesXtsDecryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 sz,
16398                           struct XtsAesStreamData *stream)
16399{
16400    int ret;
16401#ifdef WOLFSSL_AESNI
16402    Aes *aes;
16403#endif
16404
16405    if (xaes == NULL || out == NULL || in == NULL) {
16406        return BAD_FUNC_ARG;
16407    }
16408
16409#ifdef WOLFSSL_AESNI
16410#ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
16411    aes = &xaes->aes_decrypt;
16412#else
16413    aes = &xaes->aes;
16414#endif
16415#endif
16416
16417    if (sz < WC_AES_BLOCK_SIZE) {
16418        WOLFSSL_MSG("Cipher text input too small for decryption");
16419        return BAD_FUNC_ARG;
16420    }
16421
16422    if (stream->bytes_crypted_with_this_tweak &
16423        ((word32)WC_AES_BLOCK_SIZE - 1U))
16424    {
16425        WOLFSSL_MSG("AesXtsDecryptUpdate after previous finalizing call");
16426        return BAD_FUNC_ARG;
16427    }
16428
16429#ifndef WC_AESXTS_STREAM_NO_REQUEST_ACCOUNTING
16430    if (! WC_SAFE_SUM_WORD32(stream->bytes_crypted_with_this_tweak, sz,
16431                             stream->bytes_crypted_with_this_tweak))
16432    {
16433        WOLFSSL_MSG("Overflow of stream->bytes_crypted_with_this_tweak "
16434                    "in AesXtsDecryptUpdate().");
16435    }
16436#endif
16437
16438    {
16439#ifdef WOLFSSL_AESNI
16440        if (aes->use_aesni) {
16441            SAVE_VECTOR_REGISTERS(return _svr_ret;);
16442#if defined(HAVE_INTEL_AVX1)
16443            if (IS_INTEL_AVX1(intel_flags)) {
16444                AES_XTS_decrypt_update_avx1(in, out, sz,
16445                                            (const byte*)aes->key,
16446                                            stream->tweak_block,
16447                                            (int)aes->rounds);
16448                ret = 0;
16449            }
16450            else
16451#endif
16452            {
16453                AES_XTS_decrypt_update_aesni(in, out, sz,
16454                                             (const byte*)aes->key,
16455                                             stream->tweak_block,
16456                                             (int)aes->rounds);
16457                ret = 0;
16458            }
16459            RESTORE_VECTOR_REGISTERS();
16460        }
16461        else
16462#endif /* WOLFSSL_AESNI */
16463        {
16464            ret = AesXtsDecryptUpdate_sw(xaes, out, in, sz,
16465                                         stream->tweak_block);
16466        }
16467    }
16468
16469    return ret;
16470}
16471
16472int wc_AesXtsDecryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 sz,
16473                           struct XtsAesStreamData *stream)
16474{
16475    if (stream == NULL)
16476        return BAD_FUNC_ARG;
16477    if (sz & ((word32)WC_AES_BLOCK_SIZE - 1U))
16478        return BAD_FUNC_ARG;
16479    return AesXtsDecryptUpdate(xaes, out, in, sz, stream);
16480}
16481
16482int wc_AesXtsDecryptFinal(XtsAes* xaes, byte* out, const byte* in, word32 sz,
16483                           struct XtsAesStreamData *stream)
16484{
16485    int ret;
16486    if (stream == NULL)
16487        return BAD_FUNC_ARG;
16488    if (sz > 0)
16489        ret = AesXtsDecryptUpdate(xaes, out, in, sz, stream);
16490    else
16491        ret = 0;
16492    ForceZero(stream->tweak_block, WC_AES_BLOCK_SIZE);
16493    /* force the count odd, to assure error on attempt to AesXtsEncryptUpdate()
16494     * after finalization.
16495     */
16496    stream->bytes_crypted_with_this_tweak |= 1U;
16497#ifdef WOLFSSL_CHECK_MEM_ZERO
16498    wc_MemZero_Check(stream->tweak_block, WC_AES_BLOCK_SIZE);
16499#endif
16500    return ret;
16501}
16502
16503#endif /* WOLFSSL_AESXTS_STREAM */
16504#endif /* HAVE_AES_DECRYPT */
16505
16506/* Same as wc_AesXtsEncryptSector but the sector gets incremented by one every
16507 * sectorSz bytes
16508 *
16509 * xaes     AES keys to use for block encrypt
16510 * out      output buffer to hold cipher text
16511 * in       input plain text buffer to encrypt
16512 * sz       size of both out and in buffers
16513 * sector   value to use for tweak
16514 * sectorSz size of the sector
16515 *
16516 * returns 0 on success
16517 */
16518int wc_AesXtsEncryptConsecutiveSectors(XtsAes* aes, byte* out, const byte* in,
16519        word32 sz, word64 sector, word32 sectorSz)
16520{
16521    int ret  = 0;
16522    word32 iter = 0;
16523    word32 sectorCount;
16524    word32 remainder;
16525
16526    if (aes == NULL || out == NULL || in == NULL || sectorSz == 0) {
16527        return BAD_FUNC_ARG;
16528    }
16529
16530    if (sz < WC_AES_BLOCK_SIZE) {
16531        WOLFSSL_MSG("Cipher text input too small for encryption");
16532        return BAD_FUNC_ARG;
16533    }
16534
16535    sectorCount  = sz / sectorSz;
16536    remainder = sz % sectorSz;
16537
16538    while (sectorCount) {
16539        ret = wc_AesXtsEncryptSector(aes, out + (iter * sectorSz),
16540                in + (iter * sectorSz), sectorSz, sector);
16541        if (ret != 0)
16542            break;
16543
16544        sectorCount--;
16545        iter++;
16546        sector++;
16547    }
16548
16549    if (remainder && ret == 0)
16550        ret = wc_AesXtsEncryptSector(aes, out + (iter * sectorSz),
16551                in + (iter * sectorSz), remainder, sector);
16552
16553    return ret;
16554}
16555
16556#ifdef HAVE_AES_DECRYPT
16557
16558/* Same as wc_AesXtsEncryptConsecutiveSectors but Aes key is AES_DECRYPTION type
16559 *
16560 * xaes     AES keys to use for block decrypt
16561 * out      output buffer to hold cipher text
16562 * in       input plain text buffer to encrypt
16563 * sz       size of both out and in buffers
16564 * sector   value to use for tweak
16565 * sectorSz size of the sector
16566 *
16567 * returns 0 on success
16568 */
16569int wc_AesXtsDecryptConsecutiveSectors(XtsAes* aes, byte* out, const byte* in,
16570        word32 sz, word64 sector, word32 sectorSz)
16571{
16572    int ret  = 0;
16573    word32 iter = 0;
16574    word32 sectorCount;
16575    word32 remainder;
16576
16577    if (aes == NULL || out == NULL || in == NULL || sectorSz == 0) {
16578        return BAD_FUNC_ARG;
16579    }
16580
16581    if (sz < WC_AES_BLOCK_SIZE) {
16582        WOLFSSL_MSG("Cipher text input too small for decryption");
16583        return BAD_FUNC_ARG;
16584    }
16585
16586    sectorCount  = sz / sectorSz;
16587    remainder = sz % sectorSz;
16588
16589    while (sectorCount) {
16590        ret = wc_AesXtsDecryptSector(aes, out + (iter * sectorSz),
16591                in + (iter * sectorSz), sectorSz, sector);
16592        if (ret != 0)
16593            break;
16594
16595        sectorCount--;
16596        iter++;
16597        sector++;
16598    }
16599
16600    if (remainder && ret == 0)
16601        ret = wc_AesXtsDecryptSector(aes, out + (iter * sectorSz),
16602                in + (iter * sectorSz), remainder, sector);
16603
16604    return ret;
16605}
16606#endif /* HAVE_AES_DECRYPT */
16607#endif /* WOLFSSL_AES_XTS */
16608
16609#ifdef WOLFSSL_CMAC
16610
16611int wc_local_CmacUpdateAes(struct Cmac *cmac, const byte* in, word32 inSz) {
16612    int ret = 0;
16613    Aes *aes = &cmac->aes;
16614#ifdef WC_AES_HAVE_PREFETCH_ARG
16615    int did_prefetches = 0;
16616#endif
16617
16618    VECTOR_REGISTERS_PUSH;
16619
16620    while ((ret == 0) && (inSz != 0)) {
16621        word32 add = min(inSz, WC_AES_BLOCK_SIZE - cmac->bufferSz);
16622        XMEMCPY(&cmac->buffer[cmac->bufferSz], in, add);
16623
16624        cmac->bufferSz += add;
16625        inSz -= add;
16626        in += add;
16627
16628        if (cmac->bufferSz == WC_AES_BLOCK_SIZE && inSz != 0) {
16629            xorbuf(cmac->buffer, cmac->digest, WC_AES_BLOCK_SIZE);
16630            ret = AesEncrypt_preFetchOpt(aes, cmac->buffer,
16631                                            cmac->digest, &did_prefetches);
16632            if (ret == 0) {
16633                cmac->totalSz += WC_AES_BLOCK_SIZE;
16634                cmac->bufferSz = 0;
16635            }
16636        }
16637    }
16638
16639    VECTOR_REGISTERS_POP;
16640
16641    return ret;
16642}
16643
16644#endif /* WOLFSSL_CMAC */
16645
16646#ifdef WOLFSSL_AES_SIV
16647
16648/*
16649 * See RFC 5297 Section 2.4.
16650 */
16651static WARN_UNUSED_RESULT int S2V(
16652    const byte* key, word32 keySz, const AesSivAssoc* assoc, word32 numAssoc,
16653    const byte* nonce, word32 nonceSz, const byte* data,
16654    word32 dataSz, byte* out)
16655{
16656#ifdef WOLFSSL_SMALL_STACK
16657    byte* tmp[3] = {NULL, NULL, NULL};
16658    int i;
16659    Cmac* cmac;
16660#else
16661    byte tmp[3][WC_AES_BLOCK_SIZE];
16662    Cmac cmac[1];
16663#endif
16664    word32 macSz = WC_AES_BLOCK_SIZE;
16665    int ret = 0;
16666    byte tmpi = 0;
16667    word32 ai;
16668    word32 zeroBytes;
16669
16670#ifdef WOLFSSL_SMALL_STACK
16671    for (i = 0; i < 3; ++i) {
16672        tmp[i] = (byte*)XMALLOC(WC_AES_BLOCK_SIZE, NULL, DYNAMIC_TYPE_TMP_BUFFER);
16673        if (tmp[i] == NULL) {
16674            ret = MEMORY_E;
16675            break;
16676        }
16677    }
16678    if (ret == 0)
16679#endif
16680
16681    if ((numAssoc > 126) || ((nonceSz > 0) && (numAssoc > 125))) {
16682        /* See RFC 5297 Section 7. */
16683        WOLFSSL_MSG("Maximum number of ADs (including the nonce) for AES SIV is"
16684                    " 126.");
16685        ret = BAD_FUNC_ARG;
16686    }
16687
16688    if (ret == 0) {
16689        XMEMSET(tmp[1], 0, WC_AES_BLOCK_SIZE);
16690        XMEMSET(tmp[2], 0, WC_AES_BLOCK_SIZE);
16691
16692        ret = wc_AesCmacGenerate(tmp[0], &macSz, tmp[1], WC_AES_BLOCK_SIZE,
16693                                 key, keySz);
16694    }
16695
16696    if (ret == 0) {
16697        /* Loop over authenticated associated data AD1..ADn */
16698        for (ai = 0; ai < numAssoc; ++ai) {
16699            ShiftAndXorRb(tmp[1-tmpi], tmp[tmpi]);
16700            ret = wc_AesCmacGenerate(tmp[tmpi], &macSz, assoc[ai].assoc,
16701                                     assoc[ai].assocSz, key, keySz);
16702            if (ret != 0)
16703                break;
16704            xorbuf(tmp[1-tmpi], tmp[tmpi], WC_AES_BLOCK_SIZE);
16705            tmpi = (byte)(1 - tmpi);
16706        }
16707
16708        /* Add nonce as final AD. See RFC 5297 Section 3. */
16709        if ((ret == 0) && (nonceSz > 0)) {
16710            ShiftAndXorRb(tmp[1-tmpi], tmp[tmpi]);
16711            ret = wc_AesCmacGenerate(tmp[tmpi], &macSz, nonce,
16712                                     nonceSz, key, keySz);
16713            if (ret == 0) {
16714                xorbuf(tmp[1-tmpi], tmp[tmpi], WC_AES_BLOCK_SIZE);
16715            }
16716            tmpi = (byte)(1U - tmpi);
16717        }
16718
16719        /* For simplicity of the remaining code, make sure the "final" result
16720           is always in tmp[0]. */
16721        if (tmpi == 1) {
16722            XMEMCPY(tmp[0], tmp[1], WC_AES_BLOCK_SIZE);
16723        }
16724    }
16725
16726    if (ret == 0) {
16727        if (dataSz >= WC_AES_BLOCK_SIZE) {
16728
16729            WC_ALLOC_VAR_EX(cmac, Cmac, 1, NULL, DYNAMIC_TYPE_CMAC,
16730                ret=MEMORY_E);
16731            if (WC_VAR_OK(cmac))
16732            {
16733            #ifdef WOLFSSL_CHECK_MEM_ZERO
16734                /* Aes part is checked by wc_AesFree. */
16735                wc_MemZero_Add("wc_AesCmacGenerate cmac",
16736                    ((unsigned char *)cmac) + sizeof(Aes),
16737                    sizeof(Cmac) - sizeof(Aes));
16738            #endif
16739                xorbuf(tmp[0], data + (dataSz - WC_AES_BLOCK_SIZE),
16740                       WC_AES_BLOCK_SIZE);
16741                ret = wc_InitCmac(cmac, key, keySz, WC_CMAC_AES, NULL);
16742                if (ret == 0) {
16743                    ret = wc_CmacUpdate(cmac, data, dataSz - WC_AES_BLOCK_SIZE);
16744                }
16745                if (ret == 0) {
16746                    ret = wc_CmacUpdate(cmac, tmp[0], WC_AES_BLOCK_SIZE);
16747                }
16748                if (ret == 0) {
16749                    ret = wc_CmacFinal(cmac, out, &macSz);
16750                }
16751            }
16752        #ifdef WOLFSSL_SMALL_STACK
16753            XFREE(cmac, NULL, DYNAMIC_TYPE_CMAC);
16754        #elif defined(WOLFSSL_CHECK_MEM_ZERO)
16755            wc_MemZero_Check(cmac, sizeof(Cmac));
16756        #endif
16757        }
16758        else {
16759            XMEMCPY(tmp[2], data, dataSz);
16760            tmp[2][dataSz] |= 0x80;
16761            zeroBytes = WC_AES_BLOCK_SIZE - (dataSz + 1);
16762            if (zeroBytes != 0) {
16763                XMEMSET(tmp[2] + dataSz + 1, 0, zeroBytes);
16764            }
16765            ShiftAndXorRb(tmp[1], tmp[0]);
16766            xorbuf(tmp[1], tmp[2], WC_AES_BLOCK_SIZE);
16767            ret = wc_AesCmacGenerate(out, &macSz, tmp[1], WC_AES_BLOCK_SIZE, key,
16768                                     keySz);
16769        }
16770    }
16771
16772#ifdef WOLFSSL_SMALL_STACK
16773    for (i = 0; i < 3; ++i) {
16774        if (tmp[i] != NULL) {
16775            XFREE(tmp[i], NULL, DYNAMIC_TYPE_TMP_BUFFER);
16776        }
16777    }
16778#endif
16779
16780    return ret;
16781}
16782
16783static WARN_UNUSED_RESULT int AesSivCipher(
16784    const byte* key, word32 keySz, const AesSivAssoc* assoc,
16785    word32 numAssoc, const byte* nonce, word32 nonceSz,
16786    const byte* data, word32 dataSz, byte* siv, byte* out,
16787    int enc)
16788{
16789    int ret = 0;
16790    WC_DECLARE_VAR(aes, Aes, 1, 0);
16791    byte sivTmp[WC_AES_BLOCK_SIZE];
16792
16793    if (key == NULL || siv == NULL || out == NULL) {
16794        WOLFSSL_MSG("Bad parameter");
16795        ret = BAD_FUNC_ARG;
16796    }
16797
16798    if (ret == 0 && keySz != 32 && keySz != 48 && keySz != 64) {
16799        WOLFSSL_MSG("Bad key size. Must be 256, 384, or 512 bits.");
16800        ret = BAD_FUNC_ARG;
16801    }
16802
16803    if (ret == 0) {
16804        if (enc == 1) {
16805            ret = S2V(key, keySz / 2, assoc, numAssoc, nonce, nonceSz, data,
16806                      dataSz, sivTmp);
16807            if (ret != 0) {
16808                WOLFSSL_MSG("S2V failed.");
16809            }
16810            else {
16811                XMEMCPY(siv, sivTmp, WC_AES_BLOCK_SIZE);
16812            }
16813        }
16814        else {
16815            XMEMCPY(sivTmp, siv, WC_AES_BLOCK_SIZE);
16816        }
16817    }
16818
16819    if (ret == 0) {
16820#ifdef WOLFSSL_SMALL_STACK
16821        aes = wc_AesNew(NULL, INVALID_DEVID, &ret);
16822#else
16823        ret = wc_AesInit(aes, NULL, INVALID_DEVID);
16824#endif
16825        if (ret != 0) {
16826            WOLFSSL_MSG("Failed to initialized AES object.");
16827        }
16828    }
16829
16830    if (ret == 0) {
16831        if (dataSz > 0) {
16832            sivTmp[12] &= 0x7f;
16833            sivTmp[8] &= 0x7f;
16834            ret = wc_AesSetKey(aes, key + keySz / 2, keySz / 2, sivTmp,
16835                               AES_ENCRYPTION);
16836            if (ret != 0) {
16837                WOLFSSL_MSG("Failed to set key for AES-CTR.");
16838            }
16839            else {
16840                ret = wc_AesCtrEncrypt(aes, out, data, dataSz);
16841                if (ret != 0) {
16842                    WOLFSSL_MSG("AES-CTR encryption failed.");
16843                }
16844            }
16845        }
16846
16847        if (ret == 0 && enc == 0) {
16848            ret = S2V(key, keySz / 2, assoc, numAssoc, nonce, nonceSz, out,
16849                      dataSz, sivTmp);
16850            if (ret != 0) {
16851                WOLFSSL_MSG("S2V failed.");
16852            }
16853
16854            if (ConstantCompare(siv, sivTmp, WC_AES_BLOCK_SIZE) != 0) {
16855                WOLFSSL_MSG("Computed SIV doesn't match received SIV.");
16856                ret = AES_SIV_AUTH_E;
16857            }
16858        }
16859
16860    #ifdef WOLFSSL_SMALL_STACK
16861        wc_AesDelete(aes, NULL);
16862    #else
16863        wc_AesFree(aes);
16864    #endif
16865    }
16866
16867    ForceZero(sivTmp, sizeof(sivTmp));
16868
16869    return ret;
16870}
16871
16872/*
16873 * See RFC 5297 Section 2.6.
16874 */
16875int wc_AesSivEncrypt(const byte* key, word32 keySz, const byte* assoc,
16876                     word32 assocSz, const byte* nonce, word32 nonceSz,
16877                     const byte* in, word32 inSz, byte* siv, byte* out)
16878{
16879    AesSivAssoc ad;
16880    ad.assoc = assoc;
16881    ad.assocSz = assocSz;
16882    return AesSivCipher(key, keySz, &ad, 1U, nonce, nonceSz, in, inSz,
16883                        siv, out, 1);
16884}
16885
16886/*
16887 * See RFC 5297 Section 2.7.
16888 */
16889int wc_AesSivDecrypt(const byte* key, word32 keySz, const byte* assoc,
16890                     word32 assocSz, const byte* nonce, word32 nonceSz,
16891                     const byte* in, word32 inSz, byte* siv, byte* out)
16892{
16893    AesSivAssoc ad;
16894    ad.assoc = assoc;
16895    ad.assocSz = assocSz;
16896    return AesSivCipher(key, keySz, &ad, 1U, nonce, nonceSz, in, inSz,
16897                        siv, out, 0);
16898}
16899
16900/*
16901 * See RFC 5297 Section 2.6.
16902 */
16903int wc_AesSivEncrypt_ex(const byte* key, word32 keySz, const AesSivAssoc* assoc,
16904                        word32 numAssoc, const byte* nonce, word32 nonceSz,
16905                        const byte* in, word32 inSz, byte* siv, byte* out)
16906{
16907    return AesSivCipher(key, keySz, assoc, numAssoc, nonce, nonceSz, in, inSz,
16908                        siv, out, 1);
16909}
16910
16911/*
16912 * See RFC 5297 Section 2.7.
16913 */
16914int wc_AesSivDecrypt_ex(const byte* key, word32 keySz, const AesSivAssoc* assoc,
16915                        word32 numAssoc, const byte* nonce, word32 nonceSz,
16916                        const byte* in, word32 inSz, byte* siv, byte* out)
16917{
16918    return AesSivCipher(key, keySz, assoc, numAssoc, nonce, nonceSz, in, inSz,
16919                        siv, out, 0);
16920}
16921
16922#endif /* WOLFSSL_AES_SIV */
16923
16924#if defined(WOLFSSL_AES_EAX)
16925
16926/*
16927 * AES EAX one-shot API
16928 * Encrypts input data and computes an auth tag over the input
16929 * auth data and ciphertext
16930 *
16931 * Returns 0 on success
16932 * Returns error code on failure
16933 */
16934int  wc_AesEaxEncryptAuth(const byte* key, word32 keySz, byte* out,
16935                          const byte* in, word32 inSz,
16936                          const byte* nonce, word32 nonceSz,
16937                          /* output computed auth tag */
16938                          byte* authTag, word32 authTagSz,
16939                          /* input data to authenticate */
16940                          const byte* authIn, word32 authInSz)
16941{
16942#if defined(WOLFSSL_SMALL_STACK)
16943    AesEax *eax;
16944#else
16945    AesEax eax_mem;
16946    AesEax *eax = &eax_mem;
16947#endif
16948    int ret;
16949    int eaxInited = 0;
16950
16951    if (key == NULL || out == NULL || in == NULL || nonce == NULL
16952                              || authTag == NULL || authIn == NULL) {
16953        return BAD_FUNC_ARG;
16954    }
16955
16956#if defined(WOLFSSL_SMALL_STACK)
16957    if ((eax = (AesEax *)XMALLOC(sizeof(AesEax),
16958                                 NULL,
16959                                 DYNAMIC_TYPE_AES_EAX)) == NULL) {
16960        return MEMORY_E;
16961    }
16962#endif
16963
16964    if ((ret = wc_AesEaxInit(eax,
16965                             key, keySz,
16966                             nonce, nonceSz,
16967                             authIn, authInSz)) != 0) {
16968        goto cleanup;
16969    }
16970    eaxInited = 1;
16971
16972    if ((ret = wc_AesEaxEncryptUpdate(eax, out, in, inSz, NULL, 0)) != 0) {
16973        goto cleanup;
16974    }
16975
16976    if ((ret = wc_AesEaxEncryptFinal(eax, authTag, authTagSz)) != 0) {
16977        goto cleanup;
16978    }
16979
16980cleanup:
16981    if (eaxInited)
16982        wc_AesEaxFree(eax);
16983#if defined(WOLFSSL_SMALL_STACK)
16984    XFREE(eax, NULL, DYNAMIC_TYPE_AES_EAX);
16985#endif
16986    return ret;
16987}
16988
16989
16990/*
16991 * AES EAX one-shot API
16992 * Decrypts and authenticates data against a supplied auth tag
16993 *
16994 * Returns 0 on success
16995 * Returns error code on failure
16996 */
16997int  wc_AesEaxDecryptAuth(const byte* key, word32 keySz, byte* out,
16998                          const byte* in, word32 inSz,
16999                          const byte* nonce, word32 nonceSz,
17000                          /* auth tag to verify against */
17001                          const byte* authTag, word32 authTagSz,
17002                          /* input data to authenticate */
17003                          const byte* authIn, word32 authInSz)
17004{
17005#if defined(WOLFSSL_SMALL_STACK)
17006    AesEax *eax;
17007#else
17008    AesEax eax_mem;
17009    AesEax *eax = &eax_mem;
17010#endif
17011    int ret;
17012    int eaxInited = 0;
17013
17014    if (key == NULL || out == NULL || in == NULL || nonce == NULL
17015                              || authTag == NULL || authIn == NULL) {
17016        return BAD_FUNC_ARG;
17017    }
17018
17019    if (authTagSz < WOLFSSL_MIN_AUTH_TAG_SZ
17020            || authTagSz > WC_AES_BLOCK_SIZE) {
17021        return BAD_FUNC_ARG;
17022    }
17023
17024#if defined(WOLFSSL_SMALL_STACK)
17025    if ((eax = (AesEax *)XMALLOC(sizeof(AesEax),
17026                                 NULL,
17027                                 DYNAMIC_TYPE_AES_EAX)) == NULL) {
17028        return MEMORY_E;
17029    }
17030#endif
17031
17032    if ((ret = wc_AesEaxInit(eax,
17033                             key, keySz,
17034                             nonce, nonceSz,
17035                             authIn, authInSz)) != 0) {
17036
17037        goto cleanup;
17038    }
17039    eaxInited = 1;
17040
17041    if ((ret = wc_AesEaxDecryptUpdate(eax, out, in, inSz, NULL, 0)) != 0) {
17042        goto cleanup;
17043    }
17044
17045    if ((ret = wc_AesEaxDecryptFinal(eax, authTag, authTagSz)) != 0) {
17046        goto cleanup;
17047    }
17048
17049cleanup:
17050    if (eaxInited)
17051        wc_AesEaxFree(eax);
17052#if defined(WOLFSSL_SMALL_STACK)
17053    XFREE(eax, NULL, DYNAMIC_TYPE_AES_EAX);
17054#endif
17055    return ret;
17056}
17057
17058
17059/*
17060 * AES EAX Incremental API:
17061 * Initializes an AES EAX encryption or decryption operation. This must be
17062 * called before any other EAX APIs are used on the AesEax struct
17063 *
17064 * Returns 0 on success
17065 * Returns error code on failure
17066 */
17067int  wc_AesEaxInit(AesEax* eax,
17068                   const byte* key, word32 keySz,
17069                   const byte* nonce, word32 nonceSz,
17070                   const byte* authIn, word32 authInSz)
17071{
17072    int ret = 0;
17073    word32 cmacSize;
17074    int aesInited = 0;
17075    int nonceCmacInited = 0;
17076    int aadCmacInited = 0;
17077
17078    if (eax == NULL || key == NULL ||  nonce == NULL) {
17079        return BAD_FUNC_ARG;
17080    }
17081
17082    XMEMSET(eax->prefixBuf, 0, sizeof(eax->prefixBuf));
17083
17084    if ((ret = wc_AesInit(&eax->aes, NULL, INVALID_DEVID)) != 0) {
17085        goto out;
17086    }
17087    aesInited = 1;
17088
17089    if ((ret = wc_AesSetKey(&eax->aes,
17090                            key,
17091                            keySz,
17092                            NULL,
17093                            AES_ENCRYPTION)) != 0) {
17094        goto out;
17095    }
17096
17097    /*
17098    * OMAC the nonce to use as the IV for CTR encryption and auth tag chunk
17099    *   N' = OMAC^0_K(N)
17100    */
17101    if ((ret = wc_InitCmac(&eax->nonceCmac,
17102                           key,
17103                           keySz,
17104                           WC_CMAC_AES,
17105                           NULL)) != 0) {
17106        return ret;
17107    }
17108    nonceCmacInited = 1;
17109
17110    if ((ret = wc_CmacUpdate(&eax->nonceCmac,
17111                             eax->prefixBuf,
17112                             sizeof(eax->prefixBuf))) != 0) {
17113        goto out;
17114    }
17115
17116    if ((ret = wc_CmacUpdate(&eax->nonceCmac, nonce, nonceSz)) != 0) {
17117        goto out;
17118    }
17119
17120    cmacSize = WC_AES_BLOCK_SIZE;
17121    if ((ret = wc_CmacFinal(&eax->nonceCmac,
17122                            eax->nonceCmacFinal,
17123                            &cmacSize)) != 0) {
17124        goto out;
17125    }
17126
17127    if ((ret = wc_AesSetIV(&eax->aes, eax->nonceCmacFinal)) != 0) {
17128        goto out;
17129    }
17130
17131    /*
17132     * start the OMAC used to build the auth tag chunk for the AD .
17133     * This CMAC is continued in subsequent update calls when more auth data is
17134     * provided
17135     *   H' = OMAC^1_K(H)
17136     */
17137    eax->prefixBuf[WC_AES_BLOCK_SIZE-1] = 1;
17138    if ((ret = wc_InitCmac(&eax->aadCmac,
17139                           key,
17140                           keySz,
17141                           WC_CMAC_AES,
17142                           NULL)) != 0) {
17143        goto out;
17144    }
17145    aadCmacInited = 1;
17146
17147    if ((ret = wc_CmacUpdate(&eax->aadCmac,
17148                             eax->prefixBuf,
17149                             sizeof(eax->prefixBuf))) != 0) {
17150        goto out;
17151    }
17152
17153    if (authIn != NULL) {
17154        if ((ret = wc_CmacUpdate(&eax->aadCmac, authIn, authInSz)) != 0) {
17155            goto out;
17156        }
17157    }
17158
17159    /*
17160     * start the OMAC to create auth tag chunk for ciphertext. This MAC will be
17161     * updated in subsequent calls to encrypt/decrypt
17162     *  C' = OMAC^2_K(C)
17163     */
17164    eax->prefixBuf[WC_AES_BLOCK_SIZE-1] = 2;
17165    if ((ret = wc_InitCmac(&eax->ciphertextCmac,
17166                           key,
17167                           keySz,
17168                           WC_CMAC_AES,
17169                           NULL)) != 0) {
17170        goto out;
17171    }
17172
17173    if ((ret = wc_CmacUpdate(&eax->ciphertextCmac,
17174                             eax->prefixBuf,
17175                             sizeof(eax->prefixBuf))) != 0) {
17176        goto out;
17177    }
17178
17179out:
17180
17181    if (ret != 0) {
17182        if (aesInited)
17183            wc_AesFree(&eax->aes);
17184        if (nonceCmacInited)
17185            wc_CmacFree(&eax->nonceCmac);
17186        if (aadCmacInited)
17187            wc_CmacFree(&eax->aadCmac);
17188    }
17189
17190    return ret;
17191}
17192
17193
17194/*
17195 * AES EAX Incremental API:
17196 * Encrypts input plaintext using AES EAX mode, adding optional auth data to
17197 * the authentication stream
17198 *
17199 * Returns 0 on success
17200 * Returns error code on failure
17201 */
17202int  wc_AesEaxEncryptUpdate(AesEax* eax, byte* out,
17203                            const byte* in, word32 inSz,
17204                            const byte* authIn, word32 authInSz)
17205{
17206    int ret;
17207
17208    if (eax == NULL || out == NULL ||  in == NULL) {
17209        return BAD_FUNC_ARG;
17210    }
17211
17212    /*
17213     * Encrypt the plaintext using AES CTR
17214     *  C = CTR(M)
17215     */
17216    if ((ret = wc_AesCtrEncrypt(&eax->aes, out, in, inSz)) != 0) {
17217        return ret;
17218    }
17219
17220    /*
17221     * update OMAC with new ciphertext
17222     *  C' = OMAC^2_K(C)
17223     */
17224    if ((ret = wc_CmacUpdate(&eax->ciphertextCmac, out, inSz)) != 0) {
17225        return ret;
17226    }
17227
17228    /* If there exists new auth data, update the OMAC for that as well */
17229    if (authIn != NULL) {
17230        if ((ret = wc_CmacUpdate(&eax->aadCmac, authIn, authInSz)) != 0) {
17231            return ret;
17232        }
17233    }
17234
17235    return 0;
17236}
17237
17238
17239/*
17240 * AES EAX Incremental API:
17241 * Decrypts input ciphertext using AES EAX mode, adding optional auth data to
17242 * the authentication stream
17243 *
17244 * Returns 0 on success
17245 * Returns error code on failure
17246 */
17247int  wc_AesEaxDecryptUpdate(AesEax* eax, byte* out,
17248                            const byte* in, word32 inSz,
17249                            const byte* authIn, word32 authInSz)
17250{
17251    int ret;
17252
17253    if (eax == NULL || out == NULL ||  in == NULL) {
17254        return BAD_FUNC_ARG;
17255    }
17256
17257    /*
17258     * Decrypt the plaintext using AES CTR
17259     *  C = CTR(M)
17260     */
17261    if ((ret = wc_AesCtrEncrypt(&eax->aes, out, in, inSz)) != 0) {
17262        return ret;
17263    }
17264
17265    /*
17266     * update OMAC with new ciphertext
17267     *  C' = OMAC^2_K(C)
17268     */
17269    if ((ret = wc_CmacUpdate(&eax->ciphertextCmac, in, inSz)) != 0) {
17270        return ret;
17271    }
17272
17273    /* If there exists new auth data, update the OMAC for that as well */
17274    if (authIn != NULL) {
17275        if ((ret = wc_CmacUpdate(&eax->aadCmac, authIn, authInSz)) != 0) {
17276            return ret;
17277        }
17278    }
17279
17280    return 0;
17281}
17282
17283
17284/*
17285 * AES EAX Incremental API:
17286 * Provides additional auth data information to the authentication
17287 * stream for an authenticated encryption or decryption operation
17288 *
17289 * Returns 0 on success
17290 * Returns error code on failure
17291 */
17292int  wc_AesEaxAuthDataUpdate(AesEax* eax, const byte* authIn, word32 authInSz)
17293{
17294    if (eax == NULL) {
17295        return BAD_FUNC_ARG;
17296    }
17297    return wc_CmacUpdate(&eax->aadCmac, authIn, authInSz);
17298}
17299
17300
17301/*
17302 * AES EAX Incremental API:
17303 * Finalizes the authenticated encryption operation, computing the auth tag
17304 * over previously supplied auth data and computed ciphertext
17305 *
17306 * Returns 0 on success
17307 * Returns error code on failure
17308 */
17309int wc_AesEaxEncryptFinal(AesEax* eax, byte* authTag, word32 authTagSz)
17310{
17311    word32 cmacSize;
17312    int ret;
17313    word32 i;
17314
17315    if (eax == NULL || authTag == NULL || authTagSz == 0 ||
17316            authTagSz > WC_AES_BLOCK_SIZE) {
17317        return BAD_FUNC_ARG;
17318    }
17319
17320    /* Complete the OMAC for the ciphertext */
17321    cmacSize = WC_AES_BLOCK_SIZE;
17322    if ((ret = wc_CmacFinalNoFree(&eax->ciphertextCmac,
17323                                  eax->ciphertextCmacFinal,
17324                                  &cmacSize)) != 0) {
17325        return ret;
17326    }
17327
17328    /* Complete the OMAC for auth data */
17329    cmacSize = WC_AES_BLOCK_SIZE;
17330    if ((ret = wc_CmacFinalNoFree(&eax->aadCmac,
17331                                  eax->aadCmacFinal,
17332                                  &cmacSize)) != 0) {
17333        return ret;
17334    }
17335
17336    /*
17337     * Concatenate all three auth tag chunks into the final tag, truncating
17338     * at the specified tag length
17339     *   T = Tag [first authTagSz bytes]
17340     */
17341    for (i = 0; i < authTagSz; i++) {
17342        authTag[i] = eax->nonceCmacFinal[i]
17343                    ^ eax->aadCmacFinal[i]
17344                    ^ eax->ciphertextCmacFinal[i];
17345    }
17346
17347    return 0;
17348}
17349
17350
17351/*
17352 * AES EAX Incremental API:
17353 * Finalizes the authenticated decryption operation, computing the auth tag
17354 * for the previously supplied auth data and cipher text and validating it
17355 * against a provided auth tag
17356 *
17357 * Returns 0 on success
17358 * Return error code for failure
17359 */
17360int wc_AesEaxDecryptFinal(AesEax* eax,
17361                          const byte* authIn, word32 authInSz)
17362{
17363    int ret;
17364    word32 i;
17365    word32 cmacSize;
17366
17367#if defined(WOLFSSL_SMALL_STACK)
17368    byte *authTag;
17369#else
17370    byte authTag[WC_AES_BLOCK_SIZE];
17371#endif
17372
17373    if (eax == NULL || authIn == NULL || authInSz > WC_AES_BLOCK_SIZE
17374            || authInSz < WOLFSSL_MIN_AUTH_TAG_SZ) {
17375        return BAD_FUNC_ARG;
17376    }
17377
17378    /* Complete the OMAC for the ciphertext */
17379    cmacSize = WC_AES_BLOCK_SIZE;
17380    if ((ret = wc_CmacFinalNoFree(&eax->ciphertextCmac,
17381                                  eax->ciphertextCmacFinal,
17382                                  &cmacSize)) != 0) {
17383        return ret;
17384    }
17385
17386    /* Complete the OMAC for auth data */
17387    cmacSize = WC_AES_BLOCK_SIZE;
17388    if ((ret = wc_CmacFinalNoFree(&eax->aadCmac,
17389                                  eax->aadCmacFinal,
17390                                  &cmacSize)) != 0) {
17391        return ret;
17392    }
17393
17394#if defined(WOLFSSL_SMALL_STACK)
17395    authTag = (byte*)XMALLOC(WC_AES_BLOCK_SIZE, NULL, DYNAMIC_TYPE_TMP_BUFFER);
17396    if (authTag == NULL) {
17397        return MEMORY_E;
17398    }
17399#endif
17400
17401    /*
17402     * Concatenate all three auth tag chunks into the final tag, truncating
17403     * at the specified tag length
17404     *   T = Tag [first authInSz bytes]
17405     */
17406    for (i = 0; i < authInSz; i++) {
17407        authTag[i] = eax->nonceCmacFinal[i]
17408                    ^ eax->aadCmacFinal[i]
17409                    ^ eax->ciphertextCmacFinal[i];
17410    }
17411
17412    if (ConstantCompare((const byte*)authTag, authIn, (int)authInSz) != 0) {
17413        ret = AES_EAX_AUTH_E;
17414    }
17415    else {
17416        ret = 0;
17417    }
17418
17419#if defined(WOLFSSL_SMALL_STACK)
17420    XFREE(authTag, NULL, DYNAMIC_TYPE_TMP_BUFFER);
17421#endif
17422
17423    return ret;
17424}
17425
17426/*
17427 * Frees the underlying CMAC and AES contexts. Must be called when done using
17428 * the AES EAX context structure.
17429 *
17430 * Returns 0 on success
17431 * Returns error code on failure
17432 */
17433int wc_AesEaxFree(AesEax* eax)
17434{
17435    if (eax == NULL) {
17436        return BAD_FUNC_ARG;
17437    }
17438
17439    (void)wc_CmacFree(&eax->ciphertextCmac);
17440    (void)wc_CmacFree(&eax->aadCmac);
17441    wc_AesFree(&eax->aes);
17442
17443    return 0;
17444}
17445
17446#endif /* WOLFSSL_AES_EAX */
17447
17448#ifdef WOLFSSL_AES_CTS
17449
17450
17451/* One-shot API */
17452int wc_AesCtsEncrypt(const byte* key, word32 keySz, byte* out,
17453                     const byte* in, word32 inSz,
17454                     const byte* iv)
17455{
17456    WC_DECLARE_VAR(aes, Aes, 1, 0);
17457    int ret = 0;
17458    word32 outSz = inSz;
17459
17460    if (key == NULL || out == NULL || in == NULL || iv == NULL)
17461        return BAD_FUNC_ARG;
17462
17463#ifdef WOLFSSL_SMALL_STACK
17464    aes = wc_AesNew(NULL, INVALID_DEVID, &ret);
17465#else
17466    ret = wc_AesInit(aes, NULL, INVALID_DEVID);
17467#endif
17468    if (ret == 0)
17469        ret = wc_AesSetKey(aes, key, keySz, iv, AES_ENCRYPTION);
17470    if (ret == 0)
17471        ret = wc_AesCtsEncryptUpdate(aes, out, &outSz, in, inSz);
17472    if (ret == 0) {
17473        out += outSz;
17474        outSz = inSz - outSz;
17475        ret = wc_AesCtsEncryptFinal(aes, out, &outSz);
17476    }
17477
17478#ifdef WOLFSSL_SMALL_STACK
17479    wc_AesDelete(aes, NULL);
17480#else
17481    wc_AesFree(aes);
17482#endif
17483    return ret;
17484}
17485
17486int wc_AesCtsDecrypt(const byte* key, word32 keySz, byte* out,
17487                     const byte* in, word32 inSz,
17488                     const byte* iv)
17489{
17490    WC_DECLARE_VAR(aes, Aes, 1, 0);
17491    int ret = 0;
17492    word32 outSz = inSz;
17493
17494    if (key == NULL || out == NULL || in == NULL || iv == NULL) {
17495        return BAD_FUNC_ARG;
17496    }
17497
17498#ifdef WOLFSSL_SMALL_STACK
17499    aes = wc_AesNew(NULL, INVALID_DEVID, &ret);
17500#else
17501    ret = wc_AesInit(aes, NULL, INVALID_DEVID);
17502#endif
17503    if (ret == 0)
17504        ret = wc_AesSetKey(aes, key, keySz, iv, AES_DECRYPTION);
17505    if (ret == 0)
17506        ret = wc_AesCtsDecryptUpdate(aes, out, &outSz, in, inSz);
17507    if (ret == 0) {
17508        out += outSz;
17509        outSz = inSz - outSz;
17510        ret = wc_AesCtsDecryptFinal(aes, out, &outSz);
17511    }
17512
17513#ifdef WOLFSSL_SMALL_STACK
17514    wc_AesDelete(aes, NULL);
17515#else
17516    wc_AesFree(aes);
17517#endif
17518    return ret;
17519}
17520
17521static int AesCtsUpdate(Aes* aes, byte* out, word32* outSz,
17522                        const byte* in, word32 inSz, int enc)
17523{
17524    word32 blocks = 0;
17525    int ret = 0;
17526    word32 writtenSz = 0;
17527    word32 tmpOutSz;
17528
17529    if (aes == NULL || out == NULL || in == NULL || outSz == NULL)
17530        return BAD_FUNC_ARG;
17531
17532    /* Error out early for easy sanity check */
17533    if (*outSz < inSz)
17534        return BUFFER_E;
17535    tmpOutSz = *outSz;
17536
17537    /* We need to store last two blocks of plaintext */
17538    if (aes->left > 0) {
17539        word32 copySz = min(inSz, (WC_AES_BLOCK_SIZE * 2) - aes->left);
17540        XMEMCPY(aes->ctsBlock + aes->left, in, copySz);
17541        aes->left += copySz;
17542        in += copySz;
17543        inSz -= copySz;
17544
17545        if (aes->left == WC_AES_BLOCK_SIZE * 2) {
17546            if (inSz > WC_AES_BLOCK_SIZE) {
17547                if (tmpOutSz < WC_AES_BLOCK_SIZE * 2)
17548                    return BUFFER_E;
17549                if (enc) {
17550                    ret = wc_AesCbcEncrypt(aes, out, aes->ctsBlock,
17551                                           WC_AES_BLOCK_SIZE * 2);
17552                }
17553                else {
17554                    ret = wc_AesCbcDecrypt(aes, out, aes->ctsBlock,
17555                                           WC_AES_BLOCK_SIZE * 2);
17556                }
17557                if (ret != 0)
17558                    return ret;
17559                out += WC_AES_BLOCK_SIZE * 2;
17560                writtenSz += WC_AES_BLOCK_SIZE * 2;
17561                tmpOutSz -= WC_AES_BLOCK_SIZE * 2;
17562                aes->left = 0;
17563            }
17564            else if (inSz > 0) {
17565                if (tmpOutSz < WC_AES_BLOCK_SIZE)
17566                    return BUFFER_E;
17567                if (enc) {
17568                    ret = wc_AesCbcEncrypt(aes, out, aes->ctsBlock,
17569                                           WC_AES_BLOCK_SIZE);
17570                }
17571                else {
17572                    ret = wc_AesCbcDecrypt(aes, out, aes->ctsBlock,
17573                                           WC_AES_BLOCK_SIZE);
17574                }
17575                if (ret != 0)
17576                    return ret;
17577                out += WC_AES_BLOCK_SIZE;
17578                writtenSz += WC_AES_BLOCK_SIZE;
17579                tmpOutSz -= WC_AES_BLOCK_SIZE;
17580                /* Move the last block in ctsBlock to the beginning for
17581                 * next operation */
17582                XMEMCPY(aes->ctsBlock, aes->ctsBlock + WC_AES_BLOCK_SIZE,
17583                        WC_AES_BLOCK_SIZE);
17584                XMEMCPY(aes->ctsBlock + WC_AES_BLOCK_SIZE, in, inSz);
17585                aes->left = WC_AES_BLOCK_SIZE + inSz;
17586                *outSz = writtenSz;
17587                return ret; /* Return the result of encryption */
17588            }
17589            else {
17590                /* Can't output data as we need > 1 block for Final call */
17591                *outSz = writtenSz;
17592                return 0;
17593            }
17594        }
17595        else {
17596            /* All input has been absorbed into aes->ctsBlock */
17597            *outSz = 0;
17598            return 0;
17599        }
17600    }
17601    if (inSz > WC_AES_BLOCK_SIZE) {
17602        /* We need to store the last two full or partial blocks */
17603        blocks = (inSz + (WC_AES_BLOCK_SIZE - 1)) / WC_AES_BLOCK_SIZE;
17604        blocks -= 2;
17605    }
17606    if (tmpOutSz < blocks * WC_AES_BLOCK_SIZE)
17607        return BUFFER_E;
17608    if (enc)
17609        ret = wc_AesCbcEncrypt(aes, out, in, blocks * WC_AES_BLOCK_SIZE);
17610    else
17611        ret = wc_AesCbcDecrypt(aes, out, in, blocks * WC_AES_BLOCK_SIZE);
17612    in += blocks * WC_AES_BLOCK_SIZE;
17613    inSz -= blocks * WC_AES_BLOCK_SIZE;
17614    XMEMCPY(aes->ctsBlock, in, inSz);
17615    aes->left = inSz;
17616    writtenSz += blocks * WC_AES_BLOCK_SIZE;
17617    *outSz = writtenSz;
17618    return ret;
17619}
17620
17621/* Incremental API */
17622int wc_AesCtsEncryptUpdate(Aes* aes, byte* out, word32* outSz,
17623                           const byte* in, word32 inSz)
17624{
17625    return AesCtsUpdate(aes, out, outSz, in, inSz, 1);
17626}
17627
17628int wc_AesCtsEncryptFinal(Aes* aes, byte* out, word32* outSz)
17629{
17630    int ret = 0;
17631
17632    if (aes == NULL || out == NULL || outSz == NULL)
17633        return BAD_FUNC_ARG;
17634    if (*outSz < aes->left)
17635        return BUFFER_E;
17636
17637    /* Input must be at least two complete or partial blocks */
17638    if (aes->left <= WC_AES_BLOCK_SIZE)
17639        return BAD_FUNC_ARG;
17640
17641    /* Zero padding */
17642    XMEMSET(aes->ctsBlock + aes->left, 0, (WC_AES_BLOCK_SIZE * 2) - aes->left);
17643
17644    ret = wc_AesCbcEncrypt(aes, aes->ctsBlock, aes->ctsBlock,
17645                           WC_AES_BLOCK_SIZE * 2);
17646    if (ret != 0)
17647        return ret;
17648
17649    XMEMCPY(out, aes->ctsBlock + WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
17650    XMEMCPY(out + WC_AES_BLOCK_SIZE, aes->ctsBlock,
17651            aes->left - WC_AES_BLOCK_SIZE);
17652    *outSz = aes->left;
17653    return ret;
17654}
17655
17656int wc_AesCtsDecryptUpdate(Aes* aes, byte* out, word32* outSz,
17657                           const byte* in, word32 inSz)
17658{
17659    return AesCtsUpdate(aes, out, outSz, in, inSz, 0);
17660}
17661
17662int wc_AesCtsDecryptFinal(Aes* aes, byte* out, word32* outSz)
17663{
17664    int ret = 0;
17665    byte iv[WC_AES_BLOCK_SIZE];
17666    byte tmp[WC_AES_BLOCK_SIZE];
17667    word32 partialSz;
17668    word32 padSz;
17669
17670    if (aes == NULL || out == NULL || outSz == NULL)
17671        return BAD_FUNC_ARG;
17672    if (*outSz < aes->left)
17673        return BUFFER_E;
17674
17675    /* Input must be at least two complete or partial blocks */
17676    if (aes->left <= WC_AES_BLOCK_SIZE)
17677        return BAD_FUNC_ARG;
17678
17679    partialSz = aes->left - WC_AES_BLOCK_SIZE;
17680    padSz = 2 * WC_AES_BLOCK_SIZE - aes->left;
17681    /* Zero pad */
17682    XMEMSET(aes->ctsBlock + aes->left, 0, padSz);
17683
17684    /* Store IV */
17685    XMEMCPY(iv, aes->reg, WC_AES_BLOCK_SIZE);
17686    /* Load IV */
17687    XMEMCPY(aes->reg, aes->ctsBlock + WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
17688
17689    ret = wc_AesCbcDecrypt(aes, tmp, aes->ctsBlock, WC_AES_BLOCK_SIZE);
17690    if (ret != 0)
17691        return ret;
17692
17693    /* Write out partial block */
17694    XMEMCPY(out + WC_AES_BLOCK_SIZE, tmp, partialSz);
17695    /* Retrieve the padding */
17696    XMEMCPY(aes->ctsBlock + aes->left, tmp + partialSz, padSz);
17697    /* Restore IV */
17698    XMEMCPY(aes->reg, iv, WC_AES_BLOCK_SIZE);
17699
17700    ret = wc_AesCbcDecrypt(aes, out, aes->ctsBlock + WC_AES_BLOCK_SIZE,
17701                           WC_AES_BLOCK_SIZE);
17702    if (ret != 0)
17703        return ret;
17704
17705    *outSz = aes->left;
17706    return ret;
17707}
17708
17709#endif /* WOLFSSL_AES_CTS */
17710
17711#endif /* !NO_AES */