luna - wolfssl/linuxkm/lkcapi_aes

   1/* lkcapi_aes_glue.c -- glue logic for AES modes
   2 *
   3 * Copyright (C) 2006-2026 wolfSSL Inc.
   4 *
   5 * This file is part of wolfSSL.
   6 *
   7 * wolfSSL is free software; you can redistribute it and/or modify
   8 * it under the terms of the GNU General Public License as published by
   9 * the Free Software Foundation; either version 3 of the License, or
  10 * (at your option) any later version.
  11 *
  12 * wolfSSL is distributed in the hope that it will be useful,
  13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 * GNU General Public License for more details.
  16 *
  17 * You should have received a copy of the GNU General Public License
  18 * along with this program; if not, write to the Free Software
  19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
  20 */
  21
  22/* included by linuxkm/lkcapi_glue.c */
  23#ifndef WC_SKIP_INCLUDED_C_FILES
  24
  25#ifndef LINUXKM_LKCAPI_REGISTER
  26    #error lkcapi_aes_glue.c included in non-LINUXKM_LKCAPI_REGISTER project.
  27#endif
  28
  29#if (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
  30     defined(LINUXKM_LKCAPI_REGISTER_AES_ALL) || \
  31     (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_AES))) && \
  32    !defined(LINUXKM_LKCAPI_DONT_REGISTER_AES_ALL) && \
  33    !defined(LINUXKM_LKCAPI_REGISTER_AES)
  34    #define LINUXKM_LKCAPI_REGISTER_AES
  35#endif
  36
  37#if defined(LINUXKM_LKCAPI_REGISTER_AESCBC) || \
  38    defined(LINUXKM_LKCAPI_REGISTER_AESCFB) || \
  39    defined(LINUXKM_LKCAPI_REGISTER_AESGCM) || \
  40    defined(LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106) || \
  41    defined(LINUXKM_LKCAPI_REGISTER_AESCCM) || \
  42    defined(LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309) || \
  43    defined(LINUXKM_LKCAPI_REGISTER_AESXTS) || \
  44    defined(LINUXKM_LKCAPI_REGISTER_AESCTR) || \
  45    defined(LINUXKM_LKCAPI_REGISTER_AESOFB) || \
  46    defined(LINUXKM_LKCAPI_REGISTER_AESECB) || \
  47    defined(LINUXKM_LKCAPI_REGISTER_AES_ALL)
  48
  49    #ifdef NO_AES
  50        #error LINUXKM_LKCAPI_REGISTER_AES* requires !defined(NO_AES)
  51    #endif
  52
  53    #ifndef LINUXKM_LKCAPI_REGISTER_AES
  54        #define LINUXKM_LKCAPI_REGISTER_AES
  55    #endif
  56#endif
  57
  58#ifdef NO_AES
  59    #undef LINUXKM_LKCAPI_REGISTER_AES
  60#endif
  61
  62#if defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && !defined(CONFIG_CRYPTO_AES)
  63    #undef LINUXKM_LKCAPI_REGISTER_AES
  64#endif
  65
  66#if defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_AES) && \
  67    !defined(LINUXKM_LKCAPI_REGISTER_AES)
  68    #error Config conflict: CONFIG_CRYPTO_AES is defined, but LINUXKM_LKCAPI_REGISTER_AES is not.
  69#endif
  70
  71#ifdef LINUXKM_LKCAPI_REGISTER_AES
  72
  73#include <wolfssl/wolfcrypt/aes.h>
  74
  75#if defined(WC_FLAG_DONT_USE_AESNI) && !defined(WC_FLAG_DONT_USE_VECTOR_OPS)
  76    /* backward compat */
  77    #define WC_FLAG_DONT_USE_VECTOR_OPS WC_FLAG_DONT_USE_AESNI
  78#endif
  79
  80#if defined(WC_LINUXKM_C_FALLBACK_IN_SHIMS) && !defined(WC_FLAG_DONT_USE_VECTOR_OPS)
  81    #error WC_LINUXKM_C_FALLBACK_IN_SHIMS is defined but WC_FLAG_DONT_USE_VECTOR_OPS is missing.
  82#endif
  83
  84/* note the FIPS code will be returned on failure even in non-FIPS builds. */
  85#define LINUXKM_LKCAPI_AES_KAT_MISMATCH_E AES_KAT_FIPS_E
  86#define LINUXKM_LKCAPI_AESGCM_KAT_MISMATCH_E AESGCM_KAT_FIPS_E
  87
  88#define WOLFKM_AESCBC_NAME   "cbc(aes)"
  89#define WOLFKM_AESCFB_NAME   "cfb(aes)"
  90#define WOLFKM_AESGCM_NAME   "gcm(aes)"
  91#define WOLFKM_AESGCM_RFC4106_NAME   "rfc4106(gcm(aes))"
  92#define WOLFKM_AESCCM_NAME   "ccm(aes)"
  93#define WOLFKM_AESCCM_RFC4309_NAME   "rfc4309(ccm(aes))"
  94#define WOLFKM_AESXTS_NAME   "xts(aes)"
  95#define WOLFKM_AESCTR_NAME   "ctr(aes)"
  96#define WOLFKM_AESOFB_NAME   "ofb(aes)"
  97#define WOLFKM_AESECB_NAME   "ecb(aes)"
  98
  99#if defined(USE_INTEL_SPEEDUP) || defined(USE_INTEL_SPEEDUP_FOR_AES)
 100    #define WOLFKM_AES_DRIVER_ISA_EXT "-aesni-avx"
 101#elif defined(WOLFSSL_AESNI)
 102    #define WOLFKM_AES_DRIVER_ISA_EXT "-aesni"
 103#else
 104    #define WOLFKM_AES_DRIVER_ISA_EXT ""
 105#endif
 106
 107#define WOLFKM_AES_DRIVER_SUFFIX \
 108    WOLFKM_AES_DRIVER_ISA_EXT WOLFKM_DRIVER_SUFFIX_BASE
 109
 110#define WOLFKM_AESCBC_DRIVER ("cbc-aes" WOLFKM_AES_DRIVER_SUFFIX)
 111#define WOLFKM_AESCFB_DRIVER ("cfb-aes" WOLFKM_AES_DRIVER_SUFFIX)
 112#define WOLFKM_AESGCM_DRIVER ("gcm-aes" WOLFKM_AES_DRIVER_SUFFIX)
 113#define WOLFKM_AESGCM_RFC4106_DRIVER ("rfc4106-gcm-aes" WOLFKM_AES_DRIVER_SUFFIX)
 114#define WOLFKM_AESCCM_DRIVER ("ccm-aes" WOLFKM_AES_DRIVER_SUFFIX)
 115#define WOLFKM_AESCCM_RFC4309_DRIVER ("rfc4309-ccm-aes" WOLFKM_AES_DRIVER_SUFFIX)
 116#define WOLFKM_AESXTS_DRIVER ("xts-aes" WOLFKM_AES_DRIVER_SUFFIX)
 117#define WOLFKM_AESCTR_DRIVER ("ctr-aes" WOLFKM_AES_DRIVER_SUFFIX)
 118#define WOLFKM_AESOFB_DRIVER ("ofb-aes" WOLFKM_AES_DRIVER_SUFFIX)
 119#define WOLFKM_AESECB_DRIVER ("ecb-aes" WOLFKM_AES_DRIVER_SUFFIX)
 120
 121#ifdef HAVE_AES_CBC
 122    #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 123         defined(LINUXKM_LKCAPI_REGISTER_AES_ALL) || \
 124        (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_CBC))) && \
 125        !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESCBC) &&               \
 126        !defined(LINUXKM_LKCAPI_REGISTER_AESCBC)
 127        #define LINUXKM_LKCAPI_REGISTER_AESCBC
 128    #endif
 129#else
 130    #if defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_CBC) && \
 131        !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESCBC)
 132        #error Config conflict: target kernel has CONFIG_CRYPTO_CBC, but module is missing HAVE_AES_CBC.
 133    #endif
 134    #undef LINUXKM_LKCAPI_REGISTER_AESCBC
 135#endif
 136#ifdef WOLFSSL_AES_CFB
 137    #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 138         defined(LINUXKM_LKCAPI_REGISTER_AES_ALL)) && \
 139        !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESCFB) &&      \
 140        !defined(LINUXKM_LKCAPI_REGISTER_AESCFB)
 141        #define LINUXKM_LKCAPI_REGISTER_AESCFB
 142    #endif
 143#else
 144    #undef LINUXKM_LKCAPI_REGISTER_AESCFB
 145#endif
 146#ifdef HAVE_AESGCM
 147    #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 148         defined(LINUXKM_LKCAPI_REGISTER_AES_ALL) || \
 149         (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_GCM))) && \
 150        !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESGCM) &&               \
 151        !defined(LINUXKM_LKCAPI_REGISTER_AESGCM)
 152        #define LINUXKM_LKCAPI_REGISTER_AESGCM
 153    #endif
 154    #if ((defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 155          defined(LINUXKM_LKCAPI_REGISTER_AES_ALL) ||                   \
 156         (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_GCM))) && \
 157         !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESGCM_RFC4106)) &&      \
 158        !defined(LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106)
 159        #define LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106
 160    #endif
 161#else
 162    #if defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_GCM) && \
 163        !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESGCM)
 164        #error Config conflict: target kernel has CONFIG_CRYPTO_GCM, but module is missing HAVE_AESGCM.
 165    #endif
 166    #undef LINUXKM_LKCAPI_REGISTER_AESGCM
 167    #undef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106
 168#endif
 169#ifdef HAVE_AESCCM
 170    #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 171         defined(LINUXKM_LKCAPI_REGISTER_AES_ALL) || \
 172         (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_CCM))) && \
 173        !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESCCM) &&               \
 174        !defined(LINUXKM_LKCAPI_REGISTER_AESCCM)
 175        #define LINUXKM_LKCAPI_REGISTER_AESCCM
 176    #endif
 177    #if ((defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 178          defined(LINUXKM_LKCAPI_REGISTER_AES_ALL) ||                  \
 179         (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_CCM))) && \
 180        !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESCCM_RFC4309)) &&      \
 181        !defined(LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309)
 182        #define LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309
 183    #endif
 184#else
 185    #if defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_CCM) && \
 186        !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESCCM)
 187        #error Config conflict: target kernel has CONFIG_CRYPTO_CCM, but module is missing HAVE_AESCCM.
 188    #endif
 189    #undef LINUXKM_LKCAPI_REGISTER_AESCCM
 190    #undef LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309
 191#endif
 192#ifdef WOLFSSL_AES_XTS
 193    #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 194         defined(LINUXKM_LKCAPI_REGISTER_AES_ALL) || \
 195         (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_XTS))) && \
 196        !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESXTS) &&               \
 197        !defined(LINUXKM_LKCAPI_REGISTER_AESXTS)
 198        #define LINUXKM_LKCAPI_REGISTER_AESXTS
 199    #endif
 200#else
 201    #if defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_XTS) && \
 202        !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESXTS)
 203        #error Config conflict: target kernel has CONFIG_CRYPTO_XTS, but module is missing WOLFSSL_AES_XTS.
 204    #endif
 205    #undef LINUXKM_LKCAPI_REGISTER_AESXTS
 206#endif
 207#ifdef WOLFSSL_AES_COUNTER
 208    #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 209         defined(LINUXKM_LKCAPI_REGISTER_AES_ALL) || \
 210         (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_CTR))) && \
 211        !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESCTR) &&               \
 212        !defined(LINUXKM_LKCAPI_REGISTER_AESCTR)
 213        #define LINUXKM_LKCAPI_REGISTER_AESCTR
 214    #endif
 215#else
 216    #if defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_CTR) && \
 217        !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESCTR)
 218        #error Config conflict: target kernel has CONFIG_CRYPTO_CTR, but module is missing WOLFSSL_AES_COUNTER.
 219    #endif
 220    #undef LINUXKM_LKCAPI_REGISTER_AESCTR
 221#endif
 222#ifdef WOLFSSL_AES_OFB
 223    #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 224         defined(LINUXKM_LKCAPI_REGISTER_AES_ALL)) && \
 225        !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESOFB) &&      \
 226        !defined(LINUXKM_LKCAPI_REGISTER_AESOFB)
 227        #define LINUXKM_LKCAPI_REGISTER_AESOFB
 228    #endif
 229#else
 230    #undef LINUXKM_LKCAPI_REGISTER_AESOFB
 231#endif
 232#ifdef HAVE_AES_ECB
 233    #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 234         defined(LINUXKM_LKCAPI_REGISTER_AES_ALL) || \
 235         (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_ECB))) && \
 236         !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESECB) &&   \
 237        !defined(LINUXKM_LKCAPI_REGISTER_AESECB)
 238        #define LINUXKM_LKCAPI_REGISTER_AESECB
 239    #endif
 240#else
 241    #if defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_ECB) && \
 242        !defined(LINUXKM_LKCAPI_DONT_REGISTER_AESECB)
 243        #error Config conflict: target kernel has CONFIG_CRYPTO_ECB, but module is missing HAVE_AES_ECB.
 244    #endif
 245    #undef LINUXKM_LKCAPI_REGISTER_AESECB
 246#endif
 247
 248#ifdef LINUXKM_LKCAPI_REGISTER_AESCBC
 249    static int  linuxkm_test_aescbc(void);
 250#endif
 251#ifdef LINUXKM_LKCAPI_REGISTER_AESCFB
 252    static int  linuxkm_test_aescfb(void);
 253#endif
 254#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM
 255    static int  linuxkm_test_aesgcm(void);
 256#endif
 257#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106
 258    static int  linuxkm_test_aesgcm_rfc4106(void);
 259#endif
 260#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM
 261    static int  linuxkm_test_aesccm(void);
 262#endif
 263#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309
 264    static int  linuxkm_test_aesccm_rfc4309(void);
 265#endif
 266#ifdef LINUXKM_LKCAPI_REGISTER_AESXTS
 267    static int  linuxkm_test_aesxts(void);
 268#endif
 269#ifdef LINUXKM_LKCAPI_REGISTER_AESCTR
 270    static int  linuxkm_test_aesctr(void);
 271#endif
 272#ifdef LINUXKM_LKCAPI_REGISTER_AESOFB
 273    static int  linuxkm_test_aesofb(void);
 274#endif
 275#ifdef LINUXKM_LKCAPI_REGISTER_AESECB
 276    static int  linuxkm_test_aesecb(void);
 277#endif
 278
 279#if defined(LINUXKM_LKCAPI_REGISTER_AESCBC) || \
 280    defined(LINUXKM_LKCAPI_REGISTER_AESCFB) || \
 281    defined(LINUXKM_LKCAPI_REGISTER_AESCTR) || \
 282    defined(LINUXKM_LKCAPI_REGISTER_AESOFB) || \
 283    defined(LINUXKM_LKCAPI_REGISTER_AESECB) || \
 284    defined(LINUXKM_LKCAPI_REGISTER_AESGCM) || \
 285    defined(LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106) || \
 286    defined(LINUXKM_LKCAPI_REGISTER_AESCCM) || \
 287    defined(LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309)
 288    #define LINUXKM_LKCAPI_NEED_AES_COMMON_FUNCS
 289#endif
 290
 291#if defined(LINUXKM_LKCAPI_REGISTER_AESCBC) || \
 292    defined(LINUXKM_LKCAPI_REGISTER_AESCFB) || \
 293    defined(LINUXKM_LKCAPI_REGISTER_AESCTR) || \
 294    defined(LINUXKM_LKCAPI_REGISTER_AESOFB) || \
 295    defined(LINUXKM_LKCAPI_REGISTER_AESECB)
 296    #define LINUXKM_LKCAPI_NEED_AES_SKCIPHER_COMMON_FUNCS
 297#endif
 298
 299#if defined(LINUXKM_LKCAPI_REGISTER_AESGCM) || \
 300    defined(LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106) || \
 301    defined(LINUXKM_LKCAPI_REGISTER_AESCCM) || \
 302    defined(LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309)
 303    #define LINUXKM_LKCAPI_REGISTER_AEADS
 304#endif
 305
 306/* km_AesX(): wrappers to wolfcrypt wc_AesX functions and
 307 * structures.  */
 308
 309struct km_AesCtx {
 310    Aes          *aes_encrypt; /* allocated in km_AesInitCommon() to assure
 311                                * alignment, needed for AESNI.
 312                                */
 313    Aes          *aes_decrypt; /* same. */
 314#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS
 315    Aes          *aes_encrypt_C; /* fallback if vector registers aren't available. */
 316    Aes          *aes_decrypt_C;
 317#endif
 318#if defined(LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106) || \
 319    defined(LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309)
 320    union {
 321        byte rfc4106_nonce[4];
 322        byte rfc4309_nonce[3];
 323    };
 324#endif
 325};
 326
 327#ifdef LINUXKM_LKCAPI_NEED_AES_COMMON_FUNCS
 328
 329static void km_AesExitCommon(struct km_AesCtx * ctx);
 330
 331static int km_AesInitCommon(
 332    struct km_AesCtx * ctx,
 333    const char * name,
 334    int need_decryption)
 335{
 336    int err;
 337
 338    ctx->aes_encrypt = (Aes *)malloc(sizeof(*ctx->aes_encrypt));
 339
 340    if (! ctx->aes_encrypt) {
 341        pr_err("%s: allocation of %zu bytes for encryption key failed.\n",
 342               name, sizeof(*ctx->aes_encrypt));
 343        err = -ENOMEM;
 344        goto out;
 345    }
 346
 347    err = wc_AesInit(ctx->aes_encrypt, NULL, INVALID_DEVID);
 348
 349    if (unlikely(err)) {
 350        pr_err("%s: wc_AesInit failed: %d\n", name, err);
 351        free(ctx->aes_encrypt);
 352        ctx->aes_encrypt = NULL;
 353        err = -EINVAL;
 354        goto out;
 355    }
 356
 357    if (! need_decryption) {
 358        ctx->aes_decrypt = NULL;
 359    }
 360    else {
 361        ctx->aes_decrypt = (Aes *)malloc(sizeof(*ctx->aes_decrypt));
 362
 363        if (! ctx->aes_decrypt) {
 364            pr_err("%s: allocation of %zu bytes for decryption key failed.\n",
 365                   name, sizeof(*ctx->aes_decrypt));
 366            err = -ENOMEM;
 367            goto out;
 368        }
 369
 370        err = wc_AesInit(ctx->aes_decrypt, NULL, INVALID_DEVID);
 371
 372        if (unlikely(err)) {
 373            pr_err("%s: wc_AesInit failed: %d\n", name, err);
 374            free(ctx->aes_decrypt);
 375            ctx->aes_decrypt = NULL;
 376            err = -EINVAL;
 377            goto out;
 378        }
 379    }
 380
 381#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS
 382
 383    ctx->aes_encrypt_C = (Aes *)malloc(sizeof(*ctx->aes_encrypt_C));
 384
 385    if (! ctx->aes_encrypt_C) {
 386        pr_err("%s: allocation of %zu bytes for encryption key failed.\n",
 387               name, sizeof(*ctx->aes_encrypt_C));
 388        err = -ENOMEM;
 389        goto out;
 390    }
 391
 392    err = wc_AesInit(ctx->aes_encrypt_C, NULL, INVALID_DEVID);
 393
 394    if (unlikely(err)) {
 395        pr_err("%s: wc_AesInit failed: %d\n", name, err);
 396        free(ctx->aes_encrypt_C);
 397        ctx->aes_encrypt_C = NULL;
 398        err = -EINVAL;
 399        goto out;
 400    }
 401
 402    if (! need_decryption) {
 403        ctx->aes_decrypt_C = NULL;
 404    }
 405    else {
 406        ctx->aes_decrypt_C = (Aes *)malloc(sizeof(*ctx->aes_decrypt_C));
 407
 408        if (! ctx->aes_decrypt_C) {
 409            pr_err("%s: allocation of %zu bytes for decryption key failed.\n",
 410                   name, sizeof(*ctx->aes_decrypt_C));
 411            err = -MEMORY_E;
 412            goto out;
 413        }
 414
 415        err = wc_AesInit(ctx->aes_decrypt_C, NULL, INVALID_DEVID);
 416
 417        if (unlikely(err)) {
 418            pr_err("%s: wc_AesInit failed: %d\n", name, err);
 419            free(ctx->aes_decrypt_C);
 420            ctx->aes_decrypt_C = NULL;
 421            err = -EINVAL;
 422            goto out;
 423        }
 424    }
 425
 426#endif /* WC_LINUXKM_C_FALLBACK_IN_SHIMS */
 427
 428out:
 429
 430    if (err != 0)
 431        km_AesExitCommon(ctx);
 432
 433    #ifdef WOLFKM_DEBUG_AES
 434    pr_info("info: exiting km_AesInitCommon: %s: %d\n", name, err);
 435    #endif /* WOLFKM_DEBUG_AES */
 436
 437    return err;
 438}
 439
 440static int km_AesGet(struct km_AesCtx *ctx, int decrypt_p, int copy_p, Aes **aes) {
 441    Aes *ret;
 442
 443#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS
 444    /* First, check if AESNI was disabled in the main SetKey for the requested
 445     * direction.  If so, use it (the fallback schedule won't even be inited).
 446     */
 447    if (((! decrypt_p) || (! ctx->aes_decrypt)) && (! ctx->aes_encrypt->use_aesni))
 448        ret = ctx->aes_encrypt;
 449    else if (decrypt_p && ctx->aes_decrypt && (! ctx->aes_decrypt->use_aesni))
 450        ret = ctx->aes_decrypt;
 451    else if (
 452#ifdef TEST_WC_LINUXKM_C_FALLBACK_IN_SHIMS
 453             1
 454#else
 455             ! CAN_SAVE_VECTOR_REGISTERS()
 456#endif
 457        )
 458    {
 459        if (decrypt_p && ctx->aes_decrypt_C)
 460            ret = ctx->aes_decrypt_C;
 461        else
 462            ret = ctx->aes_encrypt_C;
 463        if (ret->use_aesni)
 464            return -EINVAL;
 465    }
 466    else
 467#endif /* WC_LINUXKM_C_FALLBACK_IN_SHIMS */
 468    {
 469        if (decrypt_p && ctx->aes_decrypt)
 470            ret = ctx->aes_decrypt;
 471        else
 472            ret = ctx->aes_encrypt;
 473    }
 474
 475    if (copy_p) {
 476        /* Copy the cipher state to mitigate races on Aes.reg, Aes.tmp, and dynamic Aes.use_aesni. */
 477        Aes *aes_copy = (struct Aes *)malloc(sizeof(Aes));
 478        if (aes_copy == NULL)
 479            return -ENOMEM;
 480        XMEMCPY(aes_copy, ret, sizeof(Aes));
 481#if defined(WOLFSSL_AESGCM_STREAM) && defined(WOLFSSL_SMALL_STACK) && !defined(WOLFSSL_AESNI)
 482        aes_copy->streamData = NULL;
 483#endif
 484        *aes = aes_copy;
 485    }
 486    else {
 487        *aes = ret;
 488    }
 489
 490    return 0;
 491}
 492
 493static void km_AesFree(Aes **aes) {
 494    if ((! aes) || (! *aes))
 495        return;
 496    wc_AesFree(*aes);
 497#if defined(HAVE_FIPS) && FIPS_VERSION3_LT(6,0,0)
 498    ForceZero(*aes, sizeof **aes);
 499#endif
 500    free(*aes);
 501    *aes = NULL;
 502}
 503
 504static void km_AesExitCommon(struct km_AesCtx * ctx)
 505{
 506    if (ctx->aes_encrypt) {
 507        km_AesFree(&ctx->aes_encrypt);
 508    }
 509    if (ctx->aes_decrypt) {
 510        km_AesFree(&ctx->aes_decrypt);
 511    }
 512
 513#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS
 514    if (ctx->aes_encrypt_C) {
 515        km_AesFree(&ctx->aes_encrypt_C);
 516    }
 517    if (ctx->aes_decrypt_C) {
 518        km_AesFree(&ctx->aes_decrypt_C);
 519    }
 520#endif
 521
 522    #ifdef WOLFKM_DEBUG_AES
 523    pr_info("info: exiting km_AesExitCommon\n");
 524    #endif /* WOLFKM_DEBUG_AES */
 525}
 526
 527#ifdef LINUXKM_LKCAPI_NEED_AES_SKCIPHER_COMMON_FUNCS
 528
 529static int km_AesSetKeyCommon(struct km_AesCtx * ctx, const u8 *in_key,
 530                              unsigned int key_len, const char * name)
 531{
 532    int err;
 533
 534    err = wc_AesSetKey(ctx->aes_encrypt, in_key, key_len, NULL, AES_ENCRYPTION);
 535
 536    if (unlikely(err)) {
 537        if ((! disable_setkey_warnings) && ((key_len == 16) || (key_len == 24) || (key_len == 32)))
 538            pr_err("%s: wc_AesSetKey for encryption key (len %u) failed: %d\n", name, key_len, err);
 539        return -EINVAL;
 540    }
 541
 542    if (ctx->aes_decrypt) {
 543        err = wc_AesSetKey(ctx->aes_decrypt, in_key, key_len, NULL,
 544                           AES_DECRYPTION);
 545
 546        if (unlikely(err)) {
 547            if ((! disable_setkey_warnings) && ((key_len == 16) || (key_len == 24) || (key_len == 32)))
 548                pr_err("%s: wc_AesSetKey for decryption key (len %u) failed: %d\n",
 549                       name, key_len, err);
 550            return -EINVAL;
 551        }
 552    }
 553
 554#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS
 555
 556    if (ctx->aes_encrypt->use_aesni) {
 557        ctx->aes_encrypt_C->use_aesni = WC_FLAG_DONT_USE_VECTOR_OPS;
 558
 559        err = wc_AesSetKey(ctx->aes_encrypt_C, in_key, key_len, NULL, AES_ENCRYPTION);
 560
 561        if (unlikely(err)) {
 562            if ((! disable_setkey_warnings) && ((key_len == 16) || (key_len == 24) || (key_len == 32)))
 563                pr_err("%s: wc_AesSetKey for encryption key failed: %d\n", name, err);
 564            return -EINVAL;
 565        }
 566
 567        if (ctx->aes_encrypt_C->use_aesni)
 568            pr_err("%s: after wc_AesSetKey, ctx->aes_encrypt_C has AES-NI asserted.\n", name);
 569
 570    }
 571
 572    if (ctx->aes_decrypt_C && ctx->aes_decrypt->use_aesni) {
 573        ctx->aes_decrypt_C->use_aesni = WC_FLAG_DONT_USE_VECTOR_OPS;
 574
 575        err = wc_AesSetKey(ctx->aes_decrypt_C, in_key, key_len, NULL,
 576                           AES_DECRYPTION);
 577
 578        if (unlikely(err)) {
 579            if ((! disable_setkey_warnings) && ((key_len == 16) || (key_len == 24) || (key_len == 32)))
 580                pr_err("%s: wc_AesSetKey for decryption key failed: %d\n",
 581                       name, err);
 582            return -EINVAL;
 583        }
 584
 585        if (ctx->aes_decrypt_C->use_aesni)
 586            pr_err("%s: after wc_AesSetKey, ctx->aes_decrypt_C has AES-NI asserted.\n", name);
 587    }
 588
 589#endif /* WC_LINUXKM_C_FALLBACK_IN_SHIMS */
 590
 591    #ifdef WOLFKM_DEBUG_AES
 592    pr_info("info: exiting km_AesSetKeyCommon: %s: %d\n", name, key_len);
 593    #endif /* WOLFKM_DEBUG_AES */
 594    return 0;
 595}
 596
 597static void km_AesExit(struct crypto_skcipher *tfm)
 598{
 599    struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm);
 600    km_AesExitCommon(ctx);
 601}
 602
 603#endif /* LINUXKM_LKCAPI_NEED_AES_SKCIPHER_COMMON_FUNCS */
 604
 605#endif /* LINUXKM_LKCAPI_NEED_AES_COMMON_FUNCS */
 606
 607#ifdef LINUXKM_LKCAPI_REGISTER_AESCBC
 608
 609static int km_AesCbcInit(struct crypto_skcipher *tfm)
 610{
 611    struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm);
 612    return km_AesInitCommon(ctx, WOLFKM_AESCBC_DRIVER, 1);
 613}
 614
 615static int km_AesCbcSetKey(struct crypto_skcipher *tfm, const u8 *in_key,
 616                          unsigned int key_len)
 617{
 618    struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm);
 619    return km_AesSetKeyCommon(ctx, in_key, key_len, WOLFKM_AESCBC_DRIVER);
 620}
 621
 622static int km_AesCbcEncrypt(struct skcipher_request *req)
 623{
 624    struct crypto_skcipher * tfm = NULL;
 625    struct km_AesCtx *       ctx = NULL;
 626    struct skcipher_walk     walk;
 627    unsigned int             nbytes = 0;
 628    int                      err;
 629    Aes                      *aes_copy = NULL;
 630
 631    tfm = crypto_skcipher_reqtfm(req);
 632    ctx = crypto_skcipher_ctx(tfm);
 633
 634    err = skcipher_walk_virt(&walk, req, false);
 635    if (unlikely(err)) {
 636        return err;
 637    }
 638
 639    err = km_AesGet(ctx, 0 /* decrypt_p */, 1 /* copy_p */, &aes_copy);
 640    if (unlikely(err)) {
 641        goto out;
 642    }
 643
 644    err = wc_AesSetIV(aes_copy, walk.iv);
 645
 646    if (unlikely(err)) {
 647        pr_err("%s: wc_AesSetIV failed: %d\n",
 648               crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
 649        err = -EINVAL;
 650        goto out;
 651    }
 652
 653    while ((nbytes = walk.nbytes) != 0) {
 654        err = wc_AesCbcEncrypt(aes_copy, walk.dst.virt.addr,
 655                               walk.src.virt.addr, nbytes & (~(WC_AES_BLOCK_SIZE - 1)));
 656
 657        if (unlikely(err)) {
 658            pr_err("%s: wc_AesCbcEncrypt failed for %u bytes: %d\n",
 659                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), nbytes, err);
 660            err = -EINVAL;
 661            goto out;
 662        }
 663
 664        nbytes &= WC_AES_BLOCK_SIZE - 1;
 665        err = skcipher_walk_done(&walk, nbytes);
 666    }
 667
 668    /* copy iv from wolfCrypt back to walk.iv */
 669    XMEMCPY(walk.iv, aes_copy->reg, WC_AES_BLOCK_SIZE);
 670
 671out:
 672
 673    if (err && walk.nbytes)
 674        (void)skcipher_walk_done(&walk, err);
 675
 676    km_AesFree(&aes_copy);
 677
 678    #ifdef WOLFKM_DEBUG_AES
 679    pr_info("info: exiting km_AesCbcEncrypt: err %d, cryptlen %d\n", err,
 680            req->cryptlen);
 681    #endif /* WOLFKM_DEBUG_AES */
 682
 683    return err;
 684}
 685
 686static int km_AesCbcDecrypt(struct skcipher_request *req)
 687{
 688    struct crypto_skcipher * tfm = NULL;
 689    struct km_AesCtx *       ctx = NULL;
 690    struct skcipher_walk     walk;
 691    unsigned int             nbytes = 0;
 692    int                      err;
 693    Aes                      *aes_copy = NULL;
 694
 695    tfm = crypto_skcipher_reqtfm(req);
 696    ctx = crypto_skcipher_ctx(tfm);
 697
 698    err = skcipher_walk_virt(&walk, req, false);
 699
 700    if (unlikely(err)) {
 701        return err;
 702    }
 703
 704    err = km_AesGet(ctx, 1 /* decrypt_p */, 1 /* copy_p */, &aes_copy);
 705    if (unlikely(err)) {
 706        goto out;
 707    }
 708
 709    err = wc_AesSetIV(aes_copy, walk.iv);
 710
 711    if (unlikely(err)) {
 712        if (! disable_setkey_warnings)
 713            pr_err("%s: wc_AesSetIV failed: %d\n",
 714                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
 715        err = -EINVAL;
 716        goto out;
 717    }
 718
 719    while ((nbytes = walk.nbytes) != 0) {
 720        err = wc_AesCbcDecrypt(aes_copy, walk.dst.virt.addr,
 721                               walk.src.virt.addr, nbytes & (~(WC_AES_BLOCK_SIZE - 1)));
 722
 723        if (unlikely(err)) {
 724            pr_err("%s: wc_AesCbcDecrypt failed for %u bytes: %d\n",
 725                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), nbytes, err);
 726            err = -EINVAL;
 727            goto out;
 728        }
 729
 730        nbytes &= WC_AES_BLOCK_SIZE - 1;
 731        err = skcipher_walk_done(&walk, nbytes);
 732    }
 733
 734    /* copy iv from wolfCrypt back to walk.iv */
 735    XMEMCPY(walk.iv, aes_copy->reg, WC_AES_BLOCK_SIZE);
 736
 737out:
 738
 739    if (err && walk.nbytes)
 740        (void)skcipher_walk_done(&walk, err);
 741
 742    km_AesFree(&aes_copy);
 743
 744    #ifdef WOLFKM_DEBUG_AES
 745    pr_info("info: exiting km_AesCbcDecrypt: err %d, cryptlen %d\n", err,
 746            req->cryptlen);
 747    #endif /* WOLFKM_DEBUG_AES */
 748
 749    return err;
 750}
 751
 752static struct skcipher_alg cbcAesAlg = {
 753    .base.cra_name        = WOLFKM_AESCBC_NAME,
 754    .base.cra_driver_name = WOLFKM_AESCBC_DRIVER,
 755    .base.cra_priority    = WOLFSSL_LINUXKM_LKCAPI_PRIORITY,
 756    .base.cra_blocksize   = WC_AES_BLOCK_SIZE,
 757    .base.cra_ctxsize     = sizeof(struct km_AesCtx),
 758    .base.cra_module      = THIS_MODULE,
 759    .init                 = km_AesCbcInit,
 760    .exit                 = km_AesExit,
 761    .min_keysize          = AES_128_KEY_SIZE,
 762    .max_keysize          = AES_256_KEY_SIZE,
 763    .ivsize               = WC_AES_BLOCK_SIZE,
 764    .setkey               = km_AesCbcSetKey,
 765    .encrypt              = km_AesCbcEncrypt,
 766    .decrypt              = km_AesCbcDecrypt,
 767};
 768static int cbcAesAlg_loaded = 0;
 769
 770#endif /* LINUXKM_LKCAPI_REGISTER_AESCBC */
 771
 772#ifdef LINUXKM_LKCAPI_REGISTER_AESCFB
 773
 774static int km_AesCfbInit(struct crypto_skcipher *tfm)
 775{
 776    struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm);
 777    return km_AesInitCommon(ctx, WOLFKM_AESCFB_DRIVER, 0);
 778}
 779
 780static int km_AesCfbSetKey(struct crypto_skcipher *tfm, const u8 *in_key,
 781                          unsigned int key_len)
 782{
 783    struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm);
 784    return km_AesSetKeyCommon(ctx, in_key, key_len, WOLFKM_AESCFB_DRIVER);
 785}
 786
 787static int km_AesCfbEncrypt(struct skcipher_request *req)
 788{
 789    struct crypto_skcipher * tfm = NULL;
 790    struct km_AesCtx *       ctx = NULL;
 791    struct skcipher_walk     walk;
 792    int                      err;
 793    Aes                      *aes_copy = NULL;
 794
 795    tfm = crypto_skcipher_reqtfm(req);
 796    ctx = crypto_skcipher_ctx(tfm);
 797
 798    err = skcipher_walk_virt(&walk, req, false);
 799
 800    if (unlikely(err)) {
 801        pr_err("%s: skcipher_walk_virt failed: %d\n",
 802               crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
 803        return err;
 804    }
 805
 806    err = km_AesGet(ctx, 0 /* decrypt_p */, 1 /* copy_p */, &aes_copy);
 807    if (unlikely(err)) {
 808        goto out;
 809    }
 810
 811    err = wc_AesSetIV(aes_copy, walk.iv);
 812
 813    if (unlikely(err)) {
 814        pr_err("%s: wc_AesSetIV failed: %d\n",
 815               crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
 816        err = -EINVAL;
 817        goto out;
 818    }
 819
 820    while (walk.nbytes != 0) {
 821        err = wc_AesCfbEncrypt(aes_copy, walk.dst.virt.addr,
 822                               walk.src.virt.addr, walk.nbytes);
 823
 824        if (unlikely(err)) {
 825            pr_err("%s: wc_AesCfbEncrypt failed %d\n",
 826                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
 827            err = -EINVAL;
 828            goto out;
 829        }
 830
 831        err = skcipher_walk_done(&walk, 0);
 832
 833        if (unlikely(err)) {
 834            pr_err("%s: skcipher_walk_done failed: %d\n",
 835                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
 836            goto out;
 837        }
 838    }
 839
 840    /* copy iv from wolfCrypt back to walk.iv */
 841    XMEMCPY(walk.iv, aes_copy->reg, WC_AES_BLOCK_SIZE);
 842
 843out:
 844
 845    if (err && walk.nbytes)
 846        (void)skcipher_walk_done(&walk, err);
 847
 848    km_AesFree(&aes_copy);
 849
 850    #ifdef WOLFKM_DEBUG_AES
 851    pr_info("info: exiting km_AesCfbEncrypt: err %d, cryptlen %d\n", err,
 852            req->cryptlen);
 853    #endif /* WOLFKM_DEBUG_AES */
 854
 855    return err;
 856}
 857
 858static int km_AesCfbDecrypt(struct skcipher_request *req)
 859{
 860    struct crypto_skcipher * tfm = NULL;
 861    struct km_AesCtx *       ctx = NULL;
 862    struct skcipher_walk     walk;
 863    int                      err;
 864    Aes                      *aes_copy = NULL;
 865
 866    tfm = crypto_skcipher_reqtfm(req);
 867    ctx = crypto_skcipher_ctx(tfm);
 868
 869    err = skcipher_walk_virt(&walk, req, false);
 870
 871    if (unlikely(err)) {
 872        pr_err("%s: skcipher_walk_virt failed: %d\n",
 873               crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
 874        return err;
 875    }
 876
 877    err = km_AesGet(ctx, 1 /* decrypt_p */, 1 /* copy_p */, &aes_copy);
 878    if (unlikely(err)) {
 879        goto out;
 880    }
 881
 882    err = wc_AesSetIV(aes_copy, walk.iv);
 883
 884    if (unlikely(err)) {
 885        if (! disable_setkey_warnings)
 886            pr_err("%s: wc_AesSetIV failed: %d\n",
 887                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
 888        err = -EINVAL;
 889        goto out;
 890    }
 891
 892    while (walk.nbytes != 0) {
 893        err = wc_AesCfbDecrypt(aes_copy, walk.dst.virt.addr,
 894                               walk.src.virt.addr, walk.nbytes);
 895
 896        if (unlikely(err)) {
 897            pr_err("%s: wc_AesCfbDecrypt failed: %d\n",
 898                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
 899            err = -EINVAL;
 900            goto out;
 901        }
 902
 903        err = skcipher_walk_done(&walk, 0);
 904
 905        if (unlikely(err)) {
 906            pr_err("%s: skcipher_walk_done failed: %d\n",
 907                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
 908            goto out;
 909        }
 910    }
 911
 912    /* copy iv from wolfCrypt back to walk.iv */
 913    XMEMCPY(walk.iv, aes_copy->reg, WC_AES_BLOCK_SIZE);
 914
 915out:
 916
 917    if (err && walk.nbytes)
 918        (void)skcipher_walk_done(&walk, err);
 919
 920    km_AesFree(&aes_copy);
 921
 922    #ifdef WOLFKM_DEBUG_AES
 923    pr_info("info: exiting km_AesCfbDecrypt: err %d, cryptlen %d\n", err,
 924            req->cryptlen);
 925    #endif /* WOLFKM_DEBUG_AES */
 926
 927    return err;
 928}
 929
 930static struct skcipher_alg cfbAesAlg = {
 931    .base.cra_name        = WOLFKM_AESCFB_NAME,
 932    .base.cra_driver_name = WOLFKM_AESCFB_DRIVER,
 933    .base.cra_priority    = WOLFSSL_LINUXKM_LKCAPI_PRIORITY,
 934    .base.cra_blocksize   = 1,
 935    .base.cra_ctxsize     = sizeof(struct km_AesCtx),
 936    .base.cra_module      = THIS_MODULE,
 937    .init                 = km_AesCfbInit,
 938    .exit                 = km_AesExit,
 939    .min_keysize          = AES_128_KEY_SIZE,
 940    .max_keysize          = AES_256_KEY_SIZE,
 941    .ivsize               = WC_AES_BLOCK_SIZE,
 942    .setkey               = km_AesCfbSetKey,
 943    .encrypt              = km_AesCfbEncrypt,
 944    .decrypt              = km_AesCfbDecrypt,
 945};
 946static int cfbAesAlg_loaded = 0;
 947
 948#endif /* LINUXKM_LKCAPI_REGISTER_AESCBC */
 949
 950#if defined(LINUXKM_LKCAPI_REGISTER_AESGCM) || \
 951    defined(LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106)
 952
 953static int km_AesGcmInit(struct crypto_aead * tfm)
 954{
 955    struct km_AesCtx * ctx = crypto_aead_ctx(tfm);
 956    return km_AesInitCommon(ctx, WOLFKM_AESGCM_DRIVER, 0);
 957}
 958
 959static void km_AesGcmExit(struct crypto_aead * tfm)
 960{
 961    struct km_AesCtx * ctx = crypto_aead_ctx(tfm);
 962    km_AesExitCommon(ctx);
 963}
 964
 965#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM
 966
 967static int km_AesGcmSetKey(struct crypto_aead *tfm, const u8 *in_key,
 968                           unsigned int key_len)
 969{
 970    int err;
 971    struct km_AesCtx * ctx = crypto_aead_ctx(tfm);
 972
 973    err = wc_AesGcmSetKey(ctx->aes_encrypt, in_key, key_len);
 974
 975    if (unlikely(err)) {
 976        if ((! disable_setkey_warnings) && ((key_len == 16) || (key_len == 24) || (key_len == 32)))
 977            pr_err("%s: wc_AesGcmSetKey failed: %d\n",
 978                   crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err);
 979        return -EINVAL;
 980    }
 981
 982#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS
 983    if (ctx->aes_encrypt->use_aesni) {
 984        ctx->aes_encrypt_C->use_aesni = WC_FLAG_DONT_USE_VECTOR_OPS;
 985
 986        err = wc_AesGcmSetKey(ctx->aes_encrypt_C, in_key, key_len);
 987
 988        if (unlikely(err)) {
 989            if ((! disable_setkey_warnings) && ((key_len == 16) || (key_len == 24) || (key_len == 32)))
 990                pr_err("%s: wc_AesGcmSetKey failed: %d\n",
 991                       crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err);
 992            return -EINVAL;
 993        }
 994
 995        if (ctx->aes_encrypt_C->use_aesni)
 996            pr_err("%s: after wc_AesGcmSetKey, ctx->aes_encrypt_C has AES-NI asserted.\n", WOLFKM_AESGCM_DRIVER);
 997    }
 998#endif
 999
1000    #ifdef WOLFKM_DEBUG_AES
1001    pr_info("info: exiting km_AesGcmSetKey: %d\n", key_len);
1002    #endif /* WOLFKM_DEBUG_AES */
1003    return 0;
1004}
1005
1006#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM */
1007
1008#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106
1009
1010static int km_AesGcmSetKey_Rfc4106(struct crypto_aead *tfm, const u8 *in_key,
1011                           unsigned int key_len)
1012{
1013    int err;
1014    struct km_AesCtx * ctx = crypto_aead_ctx(tfm);
1015
1016    if (key_len < 4)
1017        return -EINVAL;
1018    key_len -= 4;
1019    memcpy(ctx->rfc4106_nonce, in_key + key_len, 4);
1020
1021    err = wc_AesGcmSetKey(ctx->aes_encrypt, in_key, key_len);
1022
1023    if (unlikely(err)) {
1024        if ((! disable_setkey_warnings) && ((key_len == 16) || (key_len == 24) || (key_len == 32)))
1025            pr_err("%s: wc_AesGcmSetKey failed: %d\n",
1026                   crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err);
1027        return -EINVAL;
1028    }
1029
1030#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS
1031    if (ctx->aes_encrypt->use_aesni) {
1032        ctx->aes_encrypt_C->use_aesni = WC_FLAG_DONT_USE_VECTOR_OPS;
1033
1034        err = wc_AesGcmSetKey(ctx->aes_encrypt_C, in_key, key_len);
1035
1036        if (unlikely(err)) {
1037            if ((! disable_setkey_warnings) && ((key_len == 16) || (key_len == 24) || (key_len == 32)))
1038                pr_err("%s: wc_AesGcmSetKey failed: %d\n",
1039                       crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err);
1040            return -EINVAL;
1041        }
1042
1043        if (ctx->aes_encrypt_C->use_aesni)
1044            pr_err("%s: after wc_AesGcmSetKey, ctx->aes_encrypt_C has AES-NI asserted.\n", WOLFKM_AESGCM_DRIVER);
1045    }
1046#endif
1047
1048    #ifdef WOLFKM_DEBUG_AES
1049    pr_info("info: exiting km_AesGcmSetKey_Rfc4106: %d\n", key_len);
1050    #endif /* WOLFKM_DEBUG_AES */
1051    return 0;
1052}
1053
1054#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */
1055
1056#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM
1057
1058static int km_AesGcmSetAuthsize(struct crypto_aead *tfm, unsigned int authsize)
1059{
1060    (void)tfm;
1061
1062    switch (authsize) {
1063    case 4:
1064    case 8:
1065    case 12:
1066    case 13:
1067    case 14:
1068    case 15:
1069    case 16:
1070        return 0;
1071    }
1072
1073#ifdef WOLFSSL_LINUXKM_VERBOSE_LKCAPI_DEBUG
1074    pr_err("%s: invalid authsize: %d\n",
1075           crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), authsize);
1076#endif
1077    return -EINVAL;
1078}
1079
1080#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM */
1081
1082#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106
1083
1084static int km_AesGcmSetAuthsize_Rfc4106(struct crypto_aead *tfm, unsigned int authsize)
1085{
1086    (void)tfm;
1087
1088    switch (authsize) {
1089    case 8:
1090    case 12:
1091    case 16:
1092        return 0;
1093    }
1094
1095#ifdef WOLFSSL_LINUXKM_VERBOSE_LKCAPI_DEBUG
1096    pr_err("%s: invalid authsize: %d\n",
1097           crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), authsize);
1098#endif
1099    return -EINVAL;
1100}
1101
1102#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */
1103
1104/*
1105 * aead ciphers receive data in scatterlists in following order:
1106 *   encrypt
1107 *     req->src: aad||plaintext
1108 *     req->dst: aad||ciphertext||tag
1109 *   decrypt
1110 *     req->src: aad||ciphertext||tag
1111 *     req->dst: aad||plaintext, return 0 or -EBADMSG
1112 */
1113
1114#ifdef WOLFSSL_AESGCM_STREAM
1115
1116static int AesGcmCrypt_1(struct aead_request *req, int decrypt_p, int rfc4106_p)
1117{
1118    struct crypto_aead * tfm = NULL;
1119    struct km_AesCtx *   ctx = NULL;
1120    struct skcipher_walk walk;
1121    struct scatter_walk  assocSgWalk;
1122    u8                   authTag[WC_AES_BLOCK_SIZE];
1123    int                  err;
1124    unsigned int         assoclen = req->assoclen;
1125    u8 *                 assoc = NULL;
1126    u8 *                 assocmem = NULL;
1127    Aes                  *aes_copy = NULL;
1128
1129    tfm = crypto_aead_reqtfm(req);
1130    ctx = crypto_aead_ctx(tfm);
1131
1132    if (decrypt_p) {
1133        /* Copy out original auth tag from req->src. */
1134        scatterwalk_map_and_copy(authTag, req->src,
1135                                 req->assoclen + req->cryptlen - tfm->authsize,
1136                                 tfm->authsize, 0);
1137        err = skcipher_walk_aead_decrypt(&walk, req, false);
1138    }
1139    else {
1140        err = skcipher_walk_aead_encrypt(&walk, req, false);
1141    }
1142
1143    if (unlikely(err)) {
1144        pr_err("%s: %s failed: %d\n",
1145               crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)),
1146               decrypt_p ? "skcipher_walk_aead_decrypt" : "skcipher_walk_aead_encrypt",
1147               err);
1148        return err;
1149    }
1150
1151    err = km_AesGet(ctx, decrypt_p, 1 /* copy_p */, &aes_copy);
1152    if (unlikely(err)) {
1153        goto out;
1154    }
1155
1156#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106
1157    if (rfc4106_p) {
1158        byte rfc4106_iv[12];
1159
1160        if (unlikely(assoclen != 16 && assoclen != 20)) {
1161            err = -EINVAL;
1162            goto out;
1163        }
1164        assoclen -= 8;
1165
1166        memcpy(rfc4106_iv, ctx->rfc4106_nonce, 4);
1167        memcpy(rfc4106_iv + 4, walk.iv, 8);
1168        err = wc_AesGcmInit(aes_copy, NULL /*key*/, 0 /*keylen*/, rfc4106_iv,
1169                            GCM_NONCE_MID_SZ);
1170    }
1171    else
1172#else
1173    (void)rfc4106_p;
1174#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */
1175    {
1176        err = wc_AesGcmInit(aes_copy, NULL /*key*/, 0 /*keylen*/, walk.iv,
1177                            GCM_NONCE_MID_SZ);
1178    }
1179    if (unlikely(err)) {
1180        pr_err("%s: wc_AesGcmInit failed: %d\n",
1181               crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err);
1182        err = -EINVAL;
1183        goto out;
1184    }
1185
1186    if (req->src->length >= assoclen && req->src->length) {
1187        scatterwalk_start(&assocSgWalk, req->src);
1188#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0)
1189        scatterwalk_map(&assocSgWalk);
1190        assoc = assocSgWalk.addr;
1191#else
1192        assoc = scatterwalk_map(&assocSgWalk);
1193#endif
1194        if (unlikely(IS_ERR(assoc))) {
1195            err = (int)PTR_ERR(assoc);
1196            pr_err("%s: scatterwalk_map failed: %ld\n",
1197                   crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)),
1198                   PTR_ERR(assoc));
1199            assoc = NULL;
1200            goto out;
1201        }
1202    }
1203    else {
1204        /* assoc can be any length, so if it's noncontiguous, we have to copy it
1205         * to a contiguous heap allocation.
1206         */
1207        assocmem = malloc(assoclen);
1208        if (unlikely(assocmem == NULL)) {
1209            err = -ENOMEM;
1210            goto out;
1211        }
1212        assoc = assocmem;
1213        scatterwalk_map_and_copy(assoc, req->src, 0, assoclen, 0);
1214    }
1215
1216    if (decrypt_p) {
1217        err = wc_AesGcmDecryptUpdate(aes_copy, NULL, NULL, 0,
1218                                     assoc, assoclen);
1219    }
1220    else {
1221        err = wc_AesGcmEncryptUpdate(aes_copy, NULL, NULL, 0,
1222                                     assoc, assoclen);
1223    }
1224
1225    if (assocmem)
1226        free(assocmem);
1227    else {
1228#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0)
1229        scatterwalk_unmap(&assocSgWalk);
1230#else
1231        scatterwalk_unmap(assoc);
1232#endif
1233    }
1234
1235    if (unlikely(err)) {
1236        pr_err("%s: %s failed: %d\n",
1237               crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)),
1238               decrypt_p ? "wc_AesGcmDecryptUpdate" : "wc_AesGcmEncryptUpdate",
1239               err);
1240        err = -EINVAL;
1241        goto out;
1242    }
1243
1244    while (walk.nbytes) {
1245        if (decrypt_p) {
1246            err = wc_AesGcmDecryptUpdate(
1247                aes_copy,
1248                walk.dst.virt.addr,
1249                walk.src.virt.addr,
1250                walk.nbytes,
1251                NULL, 0);
1252        }
1253        else {
1254            err = wc_AesGcmEncryptUpdate(
1255                aes_copy,
1256                walk.dst.virt.addr,
1257                walk.src.virt.addr,
1258                walk.nbytes,
1259                NULL, 0);
1260        }
1261
1262        if (unlikely(err)) {
1263            pr_err("%s: %s failed: %d\n",
1264                   crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)),
1265                   decrypt_p ? "wc_AesGcmDecryptUpdate" : "wc_AesGcmEncryptUpdate",
1266                   err);
1267            err = -EINVAL;
1268            goto out;
1269        }
1270
1271        err = skcipher_walk_done(&walk, 0);
1272
1273        if (unlikely(err)) {
1274            pr_err("%s: skcipher_walk_done failed: %d\n",
1275                   crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err);
1276            goto out;
1277        }
1278    }
1279
1280    if (decrypt_p) {
1281        err = wc_AesGcmDecryptFinal(aes_copy, authTag, tfm->authsize);
1282        if (unlikely(err)) {
1283#ifdef WOLFSSL_LINUXKM_VERBOSE_LKCAPI_DEBUG
1284            pr_err("%s: wc_AesGcmDecryptFinal failed with return code %d\n",
1285                   crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err);
1286#endif
1287            if (err == WC_NO_ERR_TRACE(AES_GCM_AUTH_E)) {
1288                err = -EBADMSG;
1289                goto out;
1290            }
1291            else {
1292                err = -EINVAL;
1293                goto out;
1294            }
1295        }
1296    }
1297    else {
1298        err = wc_AesGcmEncryptFinal(aes_copy, authTag, tfm->authsize);
1299        if (unlikely(err)) {
1300            pr_err("%s: wc_AesGcmEncryptFinal failed with return code %d\n",
1301                   crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err);
1302            err = -EINVAL;
1303            goto out;
1304        }
1305        /* Now copy the auth tag into request scatterlist. */
1306        scatterwalk_map_and_copy(authTag, req->dst,
1307                                 req->assoclen + req->cryptlen,
1308                                 tfm->authsize, 1);
1309    }
1310
1311out:
1312
1313    if (err && walk.nbytes)
1314        (void)skcipher_walk_done(&walk, err);
1315
1316    km_AesFree(&aes_copy);
1317
1318    #ifdef WOLFKM_DEBUG_AES
1319    pr_info("info: exiting AesGcmCrypt_1: err %d, dec %d, cryptlen %d, "
1320            "assoclen %d\n", err, decrypt_p,
1321            req->cryptlen, req->assoclen);
1322    #endif /* WOLFKM_DEBUG_AES */
1323
1324    return err;
1325}
1326
1327#else /* !WOLFSSL_AESGCM_STREAM */
1328
1329static int AesGcmCrypt_1(struct aead_request *req, int decrypt_p, int rfc4106_p)
1330{
1331    struct crypto_aead * tfm = NULL;
1332    struct km_AesCtx *   ctx = NULL;
1333    struct skcipher_walk sk_walk;
1334    struct scatter_walk  in_walk, out_walk;
1335    u8                   *in_map = NULL, *out_map = NULL;
1336    u8                   authTag[WC_AES_BLOCK_SIZE];
1337    int                  err;
1338    unsigned int         assoclen = req->assoclen;
1339    u8 *                 assoc = NULL;
1340    u8 *                 sg_buf = NULL;
1341    Aes                  *aes_copy = NULL;
1342    u8 *                 in_text = NULL;
1343    u8 *                 out_text = NULL;
1344#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106
1345    byte rfc4106_iv[12];
1346#endif
1347
1348    tfm = crypto_aead_reqtfm(req);
1349    ctx = crypto_aead_ctx(tfm);
1350
1351    if (decrypt_p) {
1352        /* Copy out original auth tag from req->src. */
1353        scatterwalk_map_and_copy(authTag, req->src,
1354                                 req->assoclen + req->cryptlen - tfm->authsize,
1355                                 tfm->authsize, 0);
1356        err = skcipher_walk_aead_decrypt(&sk_walk, req, false);
1357    }
1358    else {
1359        err = skcipher_walk_aead_encrypt(&sk_walk, req, false);
1360    }
1361
1362    if (unlikely(err)) {
1363        pr_err("%s: %s failed: %d\n",
1364               crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)),
1365               decrypt_p ? "skcipher_walk_aead_decrypt" : "skcipher_walk_aead_encrypt",
1366               err);
1367        return -EINVAL;
1368    }
1369
1370    err = km_AesGet(ctx, decrypt_p, 1 /* copy_p */, &aes_copy);
1371    if (unlikely(err)) {
1372        goto out;
1373    }
1374
1375#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106
1376    if (rfc4106_p) {
1377        if (unlikely(assoclen != 16 && assoclen != 20)) {
1378            err = -EINVAL;
1379            goto out;
1380        }
1381        assoclen -= 8;
1382
1383        memcpy(rfc4106_iv, ctx->rfc4106_nonce, 4);
1384        memcpy(rfc4106_iv + 4, sk_walk.iv, 8);
1385    }
1386#else
1387    (void)rfc4106_p;
1388#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */
1389
1390    if ((req->src->length >= req->assoclen + req->cryptlen) &&
1391        (req->dst->length >= req->assoclen + req->cryptlen))
1392    {
1393        scatterwalk_start(&in_walk, req->src);
1394#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0)
1395        scatterwalk_map(&in_walk);
1396        in_map = in_walk.addr;
1397#else
1398        in_map = scatterwalk_map(&in_walk);
1399#endif
1400        if (unlikely(IS_ERR(in_map))) {
1401            err = (int)PTR_ERR(in_map);
1402            pr_err("%s: scatterwalk_map failed: %ld\n",
1403                   crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)),
1404                   PTR_ERR(in_map));
1405            in_map = NULL;
1406            goto out;
1407        }
1408        assoc = in_map;
1409        in_text = in_map + req->assoclen;
1410
1411        scatterwalk_start(&out_walk, req->dst);
1412#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0)
1413        scatterwalk_map(&out_walk);
1414        out_map = out_walk.addr;
1415#else
1416        out_map = scatterwalk_map(&out_walk);
1417#endif
1418        if (unlikely(IS_ERR(out_map))) {
1419            err = (int)PTR_ERR(out_map);
1420            pr_err("%s: scatterwalk_map failed: %ld\n",
1421                   crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)),
1422                   PTR_ERR(out_map));
1423            out_map = NULL;
1424            goto out;
1425        }
1426        out_text = out_map + req->assoclen;
1427    }
1428    else {
1429        sg_buf = malloc(req->assoclen + req->cryptlen);
1430        if (unlikely(sg_buf == NULL)) {
1431            err = -ENOMEM;
1432            goto out;
1433        }
1434        if (decrypt_p)
1435            scatterwalk_map_and_copy(sg_buf, req->src, 0, req->assoclen + req->cryptlen - tfm->authsize, 0);
1436        else
1437            scatterwalk_map_and_copy(sg_buf, req->src, 0, req->assoclen + req->cryptlen, 0);
1438        assoc = sg_buf;
1439        in_text = out_text = sg_buf + req->assoclen;
1440    }
1441
1442    if (decrypt_p) {
1443        err = wc_AesGcmDecrypt(aes_copy, out_text, in_text, req->cryptlen - tfm->authsize,
1444#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106
1445                               rfc4106_p ? rfc4106_iv :
1446#endif
1447                               sk_walk.iv, GCM_NONCE_MID_SZ,
1448                               authTag, tfm->authsize,
1449                               assoc, assoclen);
1450
1451        if (unlikely(err)) {
1452#ifdef WOLFSSL_LINUXKM_VERBOSE_LKCAPI_DEBUG
1453            pr_err("%s: wc_AesGcmDecrypt failed with return code %d\n",
1454                   crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err);
1455#endif
1456
1457            if (err == WC_NO_ERR_TRACE(AES_GCM_AUTH_E)) {
1458                err = -EBADMSG;
1459                goto out;
1460            }
1461            else {
1462                err = -EINVAL;
1463                goto out;
1464            }
1465        }
1466    }
1467    else {
1468        err = wc_AesGcmEncrypt(aes_copy, out_text, in_text, req->cryptlen,
1469#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106
1470                               rfc4106_p ? rfc4106_iv :
1471#endif
1472                               sk_walk.iv, GCM_NONCE_MID_SZ,
1473                               authTag, tfm->authsize,
1474                               assoc, assoclen);
1475
1476        if (unlikely(err)) {
1477            pr_err("%s: wc_AesGcmEncrypt failed: %d\n",
1478                   crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err);
1479            err = -EINVAL;
1480            goto out;
1481        }
1482
1483    }
1484
1485    if (sg_buf) {
1486        if (decrypt_p)
1487            scatterwalk_map_and_copy(sg_buf, req->dst, 0, req->assoclen + req->cryptlen - tfm->authsize, 1);
1488        else
1489            scatterwalk_map_and_copy(sg_buf, req->dst, 0, req->assoclen + req->cryptlen, 1);
1490    }
1491
1492    if (! decrypt_p) {
1493        /* Now copy the auth tag into request scatterlist. */
1494        scatterwalk_map_and_copy(authTag, req->dst,
1495                                 req->assoclen + req->cryptlen,
1496                                 tfm->authsize, 1);
1497    }
1498
1499out:
1500
1501    if (sg_buf) {
1502        free(sg_buf);
1503    }
1504    else {
1505        if (in_map) {
1506#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0)
1507            scatterwalk_unmap(&in_walk);
1508#else
1509            scatterwalk_unmap(in_map);
1510#endif
1511        }
1512        if (out_map) {
1513#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0)
1514            scatterwalk_unmap(&out_walk);
1515#else
1516            scatterwalk_unmap(out_map);
1517#endif
1518        }
1519    }
1520
1521    km_AesFree(&aes_copy);
1522
1523    #ifdef WOLFKM_DEBUG_AES
1524    pr_info("info: exiting AesGcmCrypt_1: err %d, dec %d, cryptlen %d, "
1525            "assoclen %d\n", err, decrypt_p,
1526            req->cryptlen, req->assoclen);
1527    #endif /* WOLFKM_DEBUG_AES */
1528
1529    return err;
1530}
1531
1532#endif /* !WOLFSSL_AESGCM_STREAM */
1533
1534#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM
1535
1536static int km_AesGcmEncrypt(struct aead_request *req) {
1537    return AesGcmCrypt_1(req, 0 /* decrypt_p */, 0 /* rfc4106_p */);
1538}
1539
1540static int km_AesGcmDecrypt(struct aead_request *req) {
1541    return AesGcmCrypt_1(req, 1 /* decrypt_p */, 0 /* rfc4106_p */);
1542}
1543
1544static struct aead_alg gcmAesAead = {
1545    .base.cra_name        = WOLFKM_AESGCM_NAME,
1546    .base.cra_driver_name = WOLFKM_AESGCM_DRIVER,
1547    .base.cra_priority    = WOLFSSL_LINUXKM_LKCAPI_PRIORITY,
1548    .base.cra_blocksize   = 1,
1549    .base.cra_ctxsize     = sizeof(struct km_AesCtx),
1550    .base.cra_module      = THIS_MODULE,
1551    .init                 = km_AesGcmInit,
1552    .exit                 = km_AesGcmExit,
1553    .setkey               = km_AesGcmSetKey,
1554    .setauthsize          = km_AesGcmSetAuthsize,
1555    .encrypt              = km_AesGcmEncrypt,
1556    .decrypt              = km_AesGcmDecrypt,
1557    .ivsize               = GCM_NONCE_MID_SZ,
1558    .maxauthsize          = WC_AES_BLOCK_SIZE,
1559    .chunksize            = WC_AES_BLOCK_SIZE,
1560};
1561static int gcmAesAead_loaded = 0;
1562
1563#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM */
1564
1565#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106
1566
1567static int km_AesGcmEncrypt_Rfc4106(struct aead_request *req) {
1568    return AesGcmCrypt_1(req, 0 /* decrypt_p */, 1 /* rfc4106 */);
1569}
1570
1571static int km_AesGcmDecrypt_Rfc4106(struct aead_request *req) {
1572    return AesGcmCrypt_1(req, 1 /* decrypt_p */, 1 /* rfc4106 */);
1573}
1574
1575static struct aead_alg gcmAesAead_rfc4106 = {
1576    .base.cra_name        = WOLFKM_AESGCM_RFC4106_NAME,
1577    .base.cra_driver_name = WOLFKM_AESGCM_RFC4106_DRIVER,
1578    .base.cra_priority    = WOLFSSL_LINUXKM_LKCAPI_PRIORITY,
1579    .base.cra_blocksize   = 1,
1580    .base.cra_ctxsize     = sizeof(struct km_AesCtx),
1581    .base.cra_module      = THIS_MODULE,
1582    .init                 = km_AesGcmInit,
1583    .exit                 = km_AesGcmExit,
1584    .setkey               = km_AesGcmSetKey_Rfc4106,
1585    .setauthsize          = km_AesGcmSetAuthsize_Rfc4106,
1586    .encrypt              = km_AesGcmEncrypt_Rfc4106,
1587    .decrypt              = km_AesGcmDecrypt_Rfc4106,
1588    .ivsize               = 8,
1589    .maxauthsize          = WC_AES_BLOCK_SIZE,
1590    .chunksize            = WC_AES_BLOCK_SIZE,
1591};
1592static int gcmAesAead_rfc4106_loaded = 0;
1593
1594#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */
1595
1596#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM || LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */
1597
1598#if defined(LINUXKM_LKCAPI_REGISTER_AESCCM) || \
1599    defined(LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309)
1600
1601static int km_AesCcmInit(struct crypto_aead * tfm)
1602{
1603    struct km_AesCtx * ctx = crypto_aead_ctx(tfm);
1604    return km_AesInitCommon(ctx, WOLFKM_AESCCM_DRIVER, 0);
1605}
1606
1607static void km_AesCcmExit(struct crypto_aead * tfm)
1608{
1609    struct km_AesCtx * ctx = crypto_aead_ctx(tfm);
1610    km_AesExitCommon(ctx);
1611}
1612
1613#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM
1614
1615static int km_AesCcmSetKey(struct crypto_aead *tfm, const u8 *in_key,
1616                           unsigned int key_len)
1617{
1618    int err;
1619    struct km_AesCtx * ctx = crypto_aead_ctx(tfm);
1620
1621    err = wc_AesCcmSetKey(ctx->aes_encrypt, in_key, key_len);
1622
1623    if (unlikely(err)) {
1624        if ((! disable_setkey_warnings) && ((key_len == 16) || (key_len == 24) || (key_len == 32)))
1625            pr_err("%s: wc_AesCcmSetKey failed: %d\n",
1626                   crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err);
1627        return -EINVAL;
1628    }
1629
1630#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS
1631    if (ctx->aes_encrypt->use_aesni) {
1632        ctx->aes_encrypt_C->use_aesni = WC_FLAG_DONT_USE_VECTOR_OPS;
1633
1634        err = wc_AesCcmSetKey(ctx->aes_encrypt_C, in_key, key_len);
1635
1636        if (unlikely(err)) {
1637            if ((! disable_setkey_warnings) && ((key_len == 16) || (key_len == 24) || (key_len == 32)))
1638                pr_err("%s: wc_AesCcmSetKey failed: %d\n",
1639                       crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err);
1640            return -EINVAL;
1641        }
1642
1643        if (ctx->aes_encrypt_C->use_aesni)
1644            pr_err("%s: after wc_AesCcmSetKey, ctx->aes_encrypt_C has AES-NI asserted.\n", WOLFKM_AESCCM_DRIVER);
1645    }
1646#endif
1647
1648    #ifdef WOLFKM_DEBUG_AES
1649    pr_info("info: exiting km_AesCcmSetKey: %d\n", key_len);
1650    #endif /* WOLFKM_DEBUG_AES */
1651    return 0;
1652}
1653
1654#endif /* LINUXKM_LKCAPI_REGISTER_AESCCM */
1655
1656#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309
1657
1658static int km_AesCcmSetKey_Rfc4309(struct crypto_aead *tfm, const u8 *in_key,
1659                                   unsigned int key_len)
1660{
1661    int err;
1662    struct km_AesCtx * ctx = crypto_aead_ctx(tfm);
1663
1664    if (key_len < 3)
1665        return -EINVAL;
1666    key_len -= 3;
1667    memcpy(ctx->rfc4309_nonce, in_key + key_len, 3);
1668
1669    err = wc_AesCcmSetKey(ctx->aes_encrypt, in_key, key_len);
1670
1671    if (unlikely(err)) {
1672        if ((! disable_setkey_warnings) && ((key_len == 16) || (key_len == 24) || (key_len == 32)))
1673            pr_err("%s: wc_AesCcmSetKey failed: %d\n",
1674                   crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err);
1675        return -EINVAL;
1676    }
1677
1678#ifdef WC_LINUXKM_C_FALLBACK_IN_SHIMS
1679    if (ctx->aes_encrypt->use_aesni) {
1680        ctx->aes_encrypt_C->use_aesni = WC_FLAG_DONT_USE_VECTOR_OPS;
1681
1682        err = wc_AesCcmSetKey(ctx->aes_encrypt_C, in_key, key_len);
1683
1684        if (unlikely(err)) {
1685            if ((! disable_setkey_warnings) && ((key_len == 16) || (key_len == 24) || (key_len == 32)))
1686                pr_err("%s: wc_AesCcmSetKey failed: %d\n",
1687                       crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err);
1688            return -EINVAL;
1689        }
1690
1691        if (ctx->aes_encrypt_C->use_aesni)
1692            pr_err("%s: after wc_AesCcmSetKey, ctx->aes_encrypt_C has AES-NI asserted.\n", WOLFKM_AESCCM_RFC4309_DRIVER);
1693    }
1694#endif
1695
1696    #ifdef WOLFKM_DEBUG_AES
1697    pr_info("info: exiting km_AesCcmSetKey_Rfc4309: %d\n", key_len);
1698    #endif /* WOLFKM_DEBUG_AES */
1699    return 0;
1700}
1701
1702#endif /* LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309 */
1703
1704#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM
1705
1706static int km_AesCcmSetAuthsize(struct crypto_aead *tfm, unsigned int authsize)
1707{
1708    (void)tfm;
1709
1710    /* RFC 3610 section 2 */
1711    switch (authsize) {
1712    case 4:
1713    case 6:
1714    case 8:
1715    case 10:
1716    case 12:
1717    case 14:
1718    case 16:
1719        return 0;
1720    }
1721
1722#ifdef WOLFSSL_LINUXKM_VERBOSE_LKCAPI_DEBUG
1723    pr_err("%s: invalid authsize: %d\n",
1724           crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), authsize);
1725#endif
1726    return -EINVAL;
1727}
1728
1729#endif /* LINUXKM_LKCAPI_REGISTER_AESCCM */
1730
1731#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309
1732
1733static int km_AesCcmSetAuthsize_Rfc4309(struct crypto_aead *tfm, unsigned int authsize)
1734{
1735    (void)tfm;
1736
1737    /* RFC 4309 permits 8, 12, and 16; the kernel rfc4309 wrapper enforces
1738     * the same set, so we match it.
1739     */
1740    switch (authsize) {
1741    case 8:
1742    case 12:
1743    case 16:
1744        return 0;
1745    }
1746
1747#ifdef WOLFSSL_LINUXKM_VERBOSE_LKCAPI_DEBUG
1748    pr_err("%s: invalid authsize: %d\n",
1749           crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), authsize);
1750#endif
1751    return -EINVAL;
1752}
1753
1754#endif /* LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309 */
1755
1756/*
1757 *   ccm(aes)             -- the generic kernel CCM convention.  ivsize is 16,
1758 *                           and req->iv is a partially-formed B0 block where
1759 *                           req->iv[0] = L - 1, req->iv[1 .. 15-L] = nonce N,
1760 *                           and bytes [16-L .. 15] are scratch (the kernel's
1761 *                           generic ccm() template would write the message
1762 *                           length there before encrypting B0).  We don't need
1763 *                           that scratch region: wc_AesCcmEncrypt rebuilds B0
1764 *                           internally from (nonce, nonceSz, inSz, ...).
1765 *                           So all we have to do is decode L from req->iv[0]
1766 *                           and present (&req->iv[1], 15 - L) as the nonce.
1767 *
1768 *   rfc4309(ccm(aes))    -- IPsec ESP convention per RFC 4309.  The trailing
1769 *                           three bytes of the key material are a per-SA salt
1770 *                           that we stash at setkey time, and ivsize is 8 (the
1771 *                           explicit per-packet IV).  At request time we form
1772 *                           an 11-byte nonce as salt(3) || req->iv(8), giving
1773 *                           L = 4, matching RFC 4309 section 4.  RFC 4309
1774 *                           also packs an extra 8 bytes (a copy of the
1775 *                           explicit IV) into the AAD region between the real
1776 *                           AAD and the ciphertext, so the real AAD length is
1777 *                           req->assoclen - 8, and the ciphertext starts at
1778 *                           offset req->assoclen.  This mirrors what the GCM
1779 *                           rfc4106 path does.
1780 *
1781 * aead ciphers receive data in scatterlists in the following order:
1782 *   encrypt
1783 *     req->src: aad||plaintext
1784 *     req->dst: aad||ciphertext||tag
1785 *   decrypt
1786 *     req->src: aad||ciphertext||tag
1787 *     req->dst: aad||plaintext, return 0 or -EBADMSG
1788 *
1789 * For rfc4309 the AAD region additionally contains an inline 8-byte copy of
1790 * the explicit IV between the real AAD and the (cipher)text, so:
1791 *     real AAD length      = req->assoclen - 8
1792 *     (cipher)text offset  = req->assoclen   (skips the inline IV too)
1793 *
1794 * Note that wc_AesCcm has no streaming API, so we always linearize the
1795 * scatterlists into a single bounce buffer when they aren't already
1796 * contiguous, exactly as the !WOLFSSL_AESGCM_STREAM branch of the GCM glue
1797 * does.
1798 */
1799
1800static int AesCcmCrypt_1(struct aead_request *req, int decrypt_p, int rfc4309_p)
1801{
1802    struct crypto_aead * tfm = NULL;
1803    struct km_AesCtx *   ctx = NULL;
1804    struct skcipher_walk sk_walk;
1805    struct scatter_walk  in_walk, out_walk;
1806    u8                   *in_map = NULL, *out_map = NULL;
1807    u8                   authTag[WC_AES_BLOCK_SIZE];
1808    int                  err;
1809    unsigned int         assoclen = req->assoclen;
1810    u8 *                 assoc = NULL;
1811    u8 *                 sg_buf = NULL;
1812    Aes                  *aes_copy = NULL;
1813    u8 *                 in_text = NULL;
1814    u8 *                 out_text = NULL;
1815    const byte *         nonce = NULL;
1816    word32               nonceSz = 0;
1817#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309
1818    byte                 rfc4309_iv[CCM_NONCE_MAX_SZ]; /* >= 11 */
1819#endif
1820
1821    tfm = crypto_aead_reqtfm(req);
1822    ctx = crypto_aead_ctx(tfm);
1823
1824    if (decrypt_p) {
1825        /* Copy out the original auth tag from req->src. */
1826        scatterwalk_map_and_copy(authTag, req->src,
1827                                 req->assoclen + req->cryptlen - tfm->authsize,
1828                                 tfm->authsize, 0);
1829        err = skcipher_walk_aead_decrypt(&sk_walk, req, false);
1830    }
1831    else {
1832        err = skcipher_walk_aead_encrypt(&sk_walk, req, false);
1833    }
1834
1835    if (unlikely(err)) {
1836        pr_err("%s: %s failed: %d\n",
1837               crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)),
1838               decrypt_p ? "skcipher_walk_aead_decrypt" : "skcipher_walk_aead_encrypt",
1839               err);
1840        return -EINVAL;
1841    }
1842
1843    err = km_AesGet(ctx, decrypt_p, 1 /* copy_p */, &aes_copy);
1844    if (unlikely(err)) {
1845        goto out;
1846    }
1847
1848#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309
1849    if (rfc4309_p) {
1850        if (unlikely(assoclen != 16 && assoclen != 20)) {
1851            err = -EINVAL;
1852            goto out;
1853        }
1854        assoclen -= 8;
1855
1856        memcpy(rfc4309_iv, ctx->rfc4309_nonce, 3);
1857        memcpy(rfc4309_iv + 3, sk_walk.iv, 8);
1858        nonce   = rfc4309_iv;
1859        nonceSz = 11;
1860    }
1861    else
1862#else
1863    (void)rfc4309_p;
1864#endif /* LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309 */
1865    {
1866        /* Generic ccm(aes): req->iv is a 16-byte buffer.
1867         *   req->iv[0]               = L - 1   (2 <= L <= 8)
1868         *   req->iv[1 .. 15-L]       = nonce N (length 15 - L = nonceSz)
1869         *   req->iv[16-L .. 15]      = scratch (we don't read it)
1870         */
1871        unsigned int L_minus_1 = ((byte *)sk_walk.iv)[0];
1872        unsigned int L         = L_minus_1 + 1U;
1873
1874        if (unlikely(L < 2U || L > 8U)) {
1875            err = -EINVAL;
1876            goto out;
1877        }
1878        nonceSz = 15U - L;
1879        nonce   = &((byte *)sk_walk.iv)[1];
1880    }
1881
1882    if ((req->src->length >= req->assoclen + req->cryptlen) &&
1883        (req->dst->length >= req->assoclen + req->cryptlen))
1884    {
1885        scatterwalk_start(&in_walk, req->src);
1886#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0)
1887        scatterwalk_map(&in_walk);
1888        in_map = in_walk.addr;
1889#else
1890        in_map = scatterwalk_map(&in_walk);
1891#endif
1892        if (unlikely(IS_ERR(in_map))) {
1893            err = (int)PTR_ERR(in_map);
1894            pr_err("%s: scatterwalk_map failed: %ld\n",
1895                   crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)),
1896                   PTR_ERR(in_map));
1897            in_map = NULL;
1898            goto out;
1899        }
1900        assoc = in_map;
1901        in_text = in_map + req->assoclen;
1902
1903        scatterwalk_start(&out_walk, req->dst);
1904#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0)
1905        scatterwalk_map(&out_walk);
1906        out_map = out_walk.addr;
1907#else
1908        out_map = scatterwalk_map(&out_walk);
1909#endif
1910        if (unlikely(IS_ERR(out_map))) {
1911            err = (int)PTR_ERR(out_map);
1912            pr_err("%s: scatterwalk_map failed: %ld\n",
1913                   crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)),
1914                   PTR_ERR(out_map));
1915            out_map = NULL;
1916            goto out;
1917        }
1918        out_text = out_map + req->assoclen;
1919    }
1920    else {
1921        sg_buf = malloc(req->assoclen + req->cryptlen);
1922        if (unlikely(sg_buf == NULL)) {
1923            err = -ENOMEM;
1924            goto out;
1925        }
1926        if (decrypt_p)
1927            scatterwalk_map_and_copy(sg_buf, req->src, 0, req->assoclen + req->cryptlen - tfm->authsize, 0);
1928        else
1929            scatterwalk_map_and_copy(sg_buf, req->src, 0, req->assoclen + req->cryptlen, 0);
1930        assoc = sg_buf;
1931        in_text = out_text = sg_buf + req->assoclen;
1932    }
1933
1934    if (decrypt_p) {
1935        err = wc_AesCcmDecrypt(aes_copy, out_text, in_text,
1936                               req->cryptlen - tfm->authsize,
1937                               nonce, nonceSz,
1938                               authTag, tfm->authsize,
1939                               assoc, assoclen);
1940
1941        if (unlikely(err)) {
1942#ifdef WOLFSSL_LINUXKM_VERBOSE_LKCAPI_DEBUG
1943            pr_err("%s: wc_AesCcmDecrypt failed with return code %d\n",
1944                   crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err);
1945#endif
1946
1947            if (err == WC_NO_ERR_TRACE(AES_CCM_AUTH_E)) {
1948                err = -EBADMSG;
1949                goto out;
1950            }
1951            else {
1952                err = -EINVAL;
1953                goto out;
1954            }
1955        }
1956    }
1957    else {
1958        err = wc_AesCcmEncrypt(aes_copy, out_text, in_text, req->cryptlen,
1959                               nonce, nonceSz,
1960                               authTag, tfm->authsize,
1961                               assoc, assoclen);
1962
1963        if (unlikely(err)) {
1964            pr_err("%s: wc_AesCcmEncrypt failed: %d\n",
1965                   crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)), err);
1966            err = -EINVAL;
1967            goto out;
1968        }
1969    }
1970
1971    if (sg_buf) {
1972        if (decrypt_p)
1973            scatterwalk_map_and_copy(sg_buf, req->dst, 0, req->assoclen + req->cryptlen - tfm->authsize, 1);
1974        else
1975            scatterwalk_map_and_copy(sg_buf, req->dst, 0, req->assoclen + req->cryptlen, 1);
1976    }
1977
1978    if (! decrypt_p) {
1979        /* Now copy the auth tag into the request scatterlist. */
1980        scatterwalk_map_and_copy(authTag, req->dst,
1981                                 req->assoclen + req->cryptlen,
1982                                 tfm->authsize, 1);
1983    }
1984
1985out:
1986
1987    if (sg_buf) {
1988        free(sg_buf);
1989    }
1990    else {
1991        if (in_map) {
1992#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0)
1993            scatterwalk_unmap(&in_walk);
1994#else
1995            scatterwalk_unmap(in_map);
1996#endif
1997        }
1998        if (out_map) {
1999#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 15, 0)
2000            scatterwalk_unmap(&out_walk);
2001#else
2002            scatterwalk_unmap(out_map);
2003#endif
2004        }
2005    }
2006
2007    km_AesFree(&aes_copy);
2008
2009    #ifdef WOLFKM_DEBUG_AES
2010    pr_info("info: exiting AesCcmCrypt_1: err %d, dec %d, cryptlen %d, "
2011            "assoclen %d\n", err, decrypt_p,
2012            req->cryptlen, req->assoclen);
2013    #endif /* WOLFKM_DEBUG_AES */
2014
2015    return err;
2016}
2017
2018#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM
2019
2020static int km_AesCcmEncrypt(struct aead_request *req) {
2021    return AesCcmCrypt_1(req, 0 /* decrypt_p */, 0 /* rfc4309_p */);
2022}
2023
2024static int km_AesCcmDecrypt(struct aead_request *req) {
2025    return AesCcmCrypt_1(req, 1 /* decrypt_p */, 0 /* rfc4309_p */);
2026}
2027
2028static struct aead_alg ccmAesAead = {
2029    .base.cra_name        = WOLFKM_AESCCM_NAME,
2030    .base.cra_driver_name = WOLFKM_AESCCM_DRIVER,
2031    .base.cra_priority    = WOLFSSL_LINUXKM_LKCAPI_PRIORITY,
2032    .base.cra_blocksize   = 1,
2033    .base.cra_ctxsize     = sizeof(struct km_AesCtx),
2034    .base.cra_module      = THIS_MODULE,
2035    .init                 = km_AesCcmInit,
2036    .exit                 = km_AesCcmExit,
2037    .setkey               = km_AesCcmSetKey,
2038    .setauthsize          = km_AesCcmSetAuthsize,
2039    .encrypt              = km_AesCcmEncrypt,
2040    .decrypt              = km_AesCcmDecrypt,
2041    .ivsize               = WC_AES_BLOCK_SIZE,
2042    .maxauthsize          = WC_AES_BLOCK_SIZE,
2043    .chunksize            = 1,
2044};
2045static int ccmAesAead_loaded = 0;
2046
2047#endif /* LINUXKM_LKCAPI_REGISTER_AESCCM */
2048
2049#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309
2050
2051static int km_AesCcmEncrypt_Rfc4309(struct aead_request *req) {
2052    return AesCcmCrypt_1(req, 0 /* decrypt_p */, 1 /* rfc4309_p */);
2053}
2054
2055static int km_AesCcmDecrypt_Rfc4309(struct aead_request *req) {
2056    return AesCcmCrypt_1(req, 1 /* decrypt_p */, 1 /* rfc4309_p */);
2057}
2058
2059static struct aead_alg ccmAesAead_rfc4309 = {
2060    .base.cra_name        = WOLFKM_AESCCM_RFC4309_NAME,
2061    .base.cra_driver_name = WOLFKM_AESCCM_RFC4309_DRIVER,
2062    .base.cra_priority    = WOLFSSL_LINUXKM_LKCAPI_PRIORITY,
2063    .base.cra_blocksize   = 1,
2064    .base.cra_ctxsize     = sizeof(struct km_AesCtx),
2065    .base.cra_module      = THIS_MODULE,
2066    .init                 = km_AesCcmInit,
2067    .exit                 = km_AesCcmExit,
2068    .setkey               = km_AesCcmSetKey_Rfc4309,
2069    .setauthsize          = km_AesCcmSetAuthsize_Rfc4309,
2070    .encrypt              = km_AesCcmEncrypt_Rfc4309,
2071    .decrypt              = km_AesCcmDecrypt_Rfc4309,
2072    .ivsize               = 8,
2073    .maxauthsize          = WC_AES_BLOCK_SIZE,
2074    .chunksize            = 1,
2075};
2076static int ccmAesAead_rfc4309_loaded = 0;
2077
2078#endif /* LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309 */
2079
2080#endif /* LINUXKM_LKCAPI_REGISTER_AESCCM || LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309 */
2081
2082
2083
2084
2085#ifdef LINUXKM_LKCAPI_REGISTER_AESXTS
2086
2087#ifndef WOLFSSL_AESXTS_STREAM
2088    #error LKCAPI registration of AES-XTS requires WOLFSSL_AESXTS_STREAM (--enable-aesxts-stream).
2089#endif
2090
2091#if defined(WOLFSSL_AESNI) && !defined(WC_C_DYNAMIC_FALLBACK)
2092    #error LKCAPI registration of AES-XTS with AESNI requires WC_C_DYNAMIC_FALLBACK.
2093#endif
2094
2095struct km_AesXtsCtx {
2096    XtsAes *aesXts; /* allocated in km_AesXtsInitCommon() to assure alignment
2097                     * for AESNI.
2098                     */
2099};
2100
2101static int km_AesXtsInitCommon(struct km_AesXtsCtx * ctx, const char * name)
2102{
2103    int err;
2104
2105    ctx->aesXts = (XtsAes *)malloc(sizeof(*ctx->aesXts));
2106
2107    if (! ctx->aesXts)
2108        return -ENOMEM;
2109
2110    err = wc_AesXtsInit(ctx->aesXts, NULL, INVALID_DEVID);
2111
2112    if (unlikely(err)) {
2113        pr_err("%s: km_AesXtsInitCommon failed: %d\n", name, err);
2114        free(ctx->aesXts);
2115        ctx->aesXts = NULL;
2116        return -EINVAL;
2117    }
2118
2119    return 0;
2120}
2121
2122static int km_AesXtsInit(struct crypto_skcipher *tfm)
2123{
2124    struct km_AesXtsCtx * ctx = crypto_skcipher_ctx(tfm);
2125    return km_AesXtsInitCommon(ctx, WOLFKM_AESXTS_DRIVER);
2126}
2127
2128static void km_AesXtsExit(struct crypto_skcipher *tfm)
2129{
2130    struct km_AesXtsCtx * ctx = crypto_skcipher_ctx(tfm);
2131    wc_AesXtsFree(ctx->aesXts);
2132    free(ctx->aesXts);
2133    ctx->aesXts = NULL;
2134}
2135
2136static int km_AesXtsSetKey(struct crypto_skcipher *tfm, const u8 *in_key,
2137                          unsigned int key_len)
2138{
2139    int err;
2140    struct km_AesXtsCtx * ctx = crypto_skcipher_ctx(tfm);
2141
2142    err = wc_AesXtsSetKeyNoInit(ctx->aesXts, in_key, key_len,
2143                                AES_ENCRYPTION_AND_DECRYPTION);
2144
2145    if (unlikely(err)) {
2146        if ((! disable_setkey_warnings) && ((key_len == 16) || (key_len == 24) || (key_len == 32)))
2147            pr_err("%s: wc_AesXtsSetKeyNoInit failed: %d\n",
2148                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2149        return -EINVAL;
2150    }
2151
2152    /* It's possible to set ctx->aesXts->{tweak,aes,aes_decrypt}.use_aesni to
2153     * WC_FLAG_DONT_USE_VECTOR_OPS here, for WC_LINUXKM_C_FALLBACK_IN_SHIMS in
2154     * AES-XTS, but we can use the WC_C_DYNAMIC_FALLBACK mechanism
2155     * unconditionally because there's no AES-XTS in Cert 4718.
2156     */
2157
2158    #ifdef WOLFKM_DEBUG_AES
2159    pr_info("info: exiting km_AesXtsSetKey: %d\n", key_len);
2160    #endif /* WOLFKM_DEBUG_AES */
2161
2162    return 0;
2163}
2164
2165/* see /usr/src/linux/drivers/md/dm-crypt.c */
2166
2167static int km_AesXtsEncrypt(struct skcipher_request *req)
2168{
2169    int                      err;
2170    struct crypto_skcipher * tfm = NULL;
2171    struct km_AesXtsCtx *    ctx = NULL;
2172    struct skcipher_walk     walk;
2173    unsigned int             nbytes = 0;
2174
2175    tfm = crypto_skcipher_reqtfm(req);
2176    ctx = crypto_skcipher_ctx(tfm);
2177
2178    if (req->cryptlen < WC_AES_BLOCK_SIZE)
2179        return -EINVAL;
2180
2181    err = skcipher_walk_virt(&walk, req, false);
2182
2183    if (unlikely(err)) {
2184        pr_err("%s: skcipher_walk_virt failed: %d\n",
2185               crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2186        return err;
2187    }
2188
2189    if (walk.nbytes == walk.total) {
2190        err = wc_AesXtsEncrypt(ctx->aesXts, walk.dst.virt.addr,
2191                               walk.src.virt.addr, walk.nbytes, walk.iv, walk.ivsize);
2192
2193        if (unlikely(err)) {
2194            pr_err("%s: wc_AesXtsEncrypt failed: %d\n",
2195                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2196            return -EINVAL;
2197        }
2198
2199        err = skcipher_walk_done(&walk, 0);
2200
2201    } else {
2202        int tail = req->cryptlen % WC_AES_BLOCK_SIZE;
2203        struct skcipher_request subreq;
2204        struct XtsAesStreamData stream;
2205
2206        if (tail > 0) {
2207            int blocks = DIV_ROUND_UP(req->cryptlen, WC_AES_BLOCK_SIZE) - 2;
2208
2209            skcipher_walk_abort(&walk);
2210
2211            skcipher_request_set_tfm(&subreq, tfm);
2212            skcipher_request_set_callback(&subreq,
2213                                          skcipher_request_flags(req),
2214                                          NULL, NULL);
2215            skcipher_request_set_crypt(&subreq, req->src, req->dst,
2216                                       blocks * WC_AES_BLOCK_SIZE, req->iv);
2217            req = &subreq;
2218
2219            err = skcipher_walk_virt(&walk, req, false);
2220            if (!walk.nbytes)
2221                return err ? : -EINVAL;
2222        } else {
2223            tail = 0;
2224        }
2225
2226        err = wc_AesXtsEncryptInit(ctx->aesXts, walk.iv, walk.ivsize, &stream);
2227
2228        if (unlikely(err)) {
2229            pr_err("%s: wc_AesXtsEncryptInit failed: %d\n",
2230                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2231            return -EINVAL;
2232        }
2233
2234        while ((nbytes = walk.nbytes) != 0) {
2235            /* if this isn't the final call, pass block-aligned data to prevent
2236             * end-of-message ciphertext stealing.
2237             */
2238            if (nbytes < walk.total)
2239                nbytes &= ~(WC_AES_BLOCK_SIZE - 1);
2240
2241            if (nbytes & ((unsigned int)WC_AES_BLOCK_SIZE - 1U))
2242                err = wc_AesXtsEncryptFinal(ctx->aesXts, walk.dst.virt.addr,
2243                                            walk.src.virt.addr, nbytes,
2244                                            &stream);
2245            else
2246                err = wc_AesXtsEncryptUpdate(ctx->aesXts, walk.dst.virt.addr,
2247                                             walk.src.virt.addr, nbytes,
2248                                             &stream);
2249
2250            if (unlikely(err)) {
2251                pr_err("%s: wc_AesXtsEncryptUpdate failed: %d\n",
2252                       crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2253                return -EINVAL;
2254            }
2255
2256            err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
2257
2258            if (unlikely(err)) {
2259                pr_err("%s: skcipher_walk_done failed: %d\n",
2260                       crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2261                return err;
2262            }
2263        }
2264
2265        if (unlikely(tail > 0)) {
2266            struct scatterlist sg_src[2], sg_dst[2];
2267            struct scatterlist *src, *dst;
2268
2269            dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
2270            if (req->dst != req->src)
2271                dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
2272
2273            skcipher_request_set_crypt(req, src, dst, WC_AES_BLOCK_SIZE + tail,
2274                                       req->iv);
2275
2276            err = skcipher_walk_virt(&walk, &subreq, false);
2277            if (err)
2278                return err;
2279
2280            err = wc_AesXtsEncryptFinal(ctx->aesXts, walk.dst.virt.addr,
2281                                         walk.src.virt.addr, walk.nbytes,
2282                                         &stream);
2283
2284            if (unlikely(err)) {
2285                pr_err("%s: wc_AesXtsEncryptFinal failed: %d\n",
2286                       crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2287                return -EINVAL;
2288            }
2289
2290            err = skcipher_walk_done(&walk, 0);
2291        } else if (! (stream.bytes_crypted_with_this_tweak & ((word32)WC_AES_BLOCK_SIZE - 1U))) {
2292            err = wc_AesXtsEncryptFinal(ctx->aesXts, NULL, NULL, 0, &stream);
2293        }
2294    }
2295
2296    #ifdef WOLFKM_DEBUG_AES
2297    pr_info("info: exiting km_AesXtsEncrypt: err %d, cryptlen %d\n", err,
2298            req->cryptlen);
2299    #endif /* WOLFKM_DEBUG_AES */
2300
2301    return err;
2302}
2303
2304static int km_AesXtsDecrypt(struct skcipher_request *req)
2305{
2306    int                      err;
2307    struct crypto_skcipher * tfm = NULL;
2308    struct km_AesXtsCtx *    ctx = NULL;
2309    struct skcipher_walk     walk;
2310    unsigned int             nbytes = 0;
2311
2312    tfm = crypto_skcipher_reqtfm(req);
2313    ctx = crypto_skcipher_ctx(tfm);
2314
2315    if (req->cryptlen < WC_AES_BLOCK_SIZE)
2316        return -EINVAL;
2317
2318    err = skcipher_walk_virt(&walk, req, false);
2319
2320    if (unlikely(err)) {
2321        pr_err("%s: skcipher_walk_virt failed: %d\n",
2322               crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2323        return err;
2324    }
2325
2326    if (walk.nbytes == walk.total) {
2327        err = wc_AesXtsDecrypt(ctx->aesXts,
2328                               walk.dst.virt.addr, walk.src.virt.addr,
2329                               walk.nbytes, walk.iv, walk.ivsize);
2330
2331        if (unlikely(err)) {
2332            pr_err("%s: wc_AesXtsDecrypt failed: %d\n",
2333                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2334            return -EINVAL;
2335        }
2336
2337        err = skcipher_walk_done(&walk, 0);
2338    } else {
2339        int tail = req->cryptlen % WC_AES_BLOCK_SIZE;
2340        struct skcipher_request subreq;
2341        struct XtsAesStreamData stream;
2342
2343        if (unlikely(tail > 0)) {
2344            int blocks = DIV_ROUND_UP(req->cryptlen, WC_AES_BLOCK_SIZE) - 2;
2345
2346            skcipher_walk_abort(&walk);
2347
2348            skcipher_request_set_tfm(&subreq, tfm);
2349            skcipher_request_set_callback(&subreq,
2350                                          skcipher_request_flags(req),
2351                                          NULL, NULL);
2352            skcipher_request_set_crypt(&subreq, req->src, req->dst,
2353                                       blocks * WC_AES_BLOCK_SIZE, req->iv);
2354            req = &subreq;
2355
2356            err = skcipher_walk_virt(&walk, req, false);
2357            if (!walk.nbytes)
2358                return err ? : -EINVAL;
2359        } else {
2360            tail = 0;
2361        }
2362
2363        err = wc_AesXtsDecryptInit(ctx->aesXts, walk.iv, walk.ivsize, &stream);
2364
2365        if (unlikely(err)) {
2366            pr_err("%s: wc_AesXtsDecryptInit failed: %d\n",
2367                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2368            return -EINVAL;
2369        }
2370
2371        while ((nbytes = walk.nbytes) != 0) {
2372            /* if this isn't the final call, pass block-aligned data to prevent
2373             * end-of-message ciphertext stealing.
2374             */
2375            if (nbytes < walk.total)
2376                nbytes &= ~(WC_AES_BLOCK_SIZE - 1);
2377
2378            if (nbytes & ((unsigned int)WC_AES_BLOCK_SIZE - 1U))
2379                err = wc_AesXtsDecryptFinal(ctx->aesXts, walk.dst.virt.addr,
2380                                            walk.src.virt.addr, nbytes,
2381                                            &stream);
2382            else
2383                err = wc_AesXtsDecryptUpdate(ctx->aesXts, walk.dst.virt.addr,
2384                                             walk.src.virt.addr, nbytes,
2385                                             &stream);
2386
2387            if (unlikely(err)) {
2388                pr_err("%s: wc_AesXtsDecryptUpdate failed: %d\n",
2389                       crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2390                return -EINVAL;
2391            }
2392
2393            err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
2394
2395            if (unlikely(err)) {
2396                pr_err("%s: skcipher_walk_done failed: %d\n",
2397                       crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2398                return err;
2399            }
2400        }
2401
2402        if (unlikely(tail > 0)) {
2403            struct scatterlist sg_src[2], sg_dst[2];
2404            struct scatterlist *src, *dst;
2405
2406            dst = src = scatterwalk_ffwd(sg_src, req->src, req->cryptlen);
2407            if (req->dst != req->src)
2408                dst = scatterwalk_ffwd(sg_dst, req->dst, req->cryptlen);
2409
2410            skcipher_request_set_crypt(req, src, dst, WC_AES_BLOCK_SIZE + tail,
2411                                       req->iv);
2412
2413            err = skcipher_walk_virt(&walk, &subreq, false);
2414            if (err)
2415                return err;
2416
2417            err = wc_AesXtsDecryptFinal(ctx->aesXts, walk.dst.virt.addr,
2418                                         walk.src.virt.addr, walk.nbytes,
2419                                         &stream);
2420
2421            if (unlikely(err)) {
2422                pr_err("%s: wc_AesXtsDecryptFinal failed: %d\n",
2423                       crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2424                return -EINVAL;
2425            }
2426
2427            err = skcipher_walk_done(&walk, 0);
2428        } else if (! (stream.bytes_crypted_with_this_tweak & ((word32)WC_AES_BLOCK_SIZE - 1U))) {
2429            err = wc_AesXtsDecryptFinal(ctx->aesXts, NULL, NULL, 0, &stream);
2430        }
2431    }
2432
2433    #ifdef WOLFKM_DEBUG_AES
2434    pr_info("info: exiting km_AesXtsDecrypt: err %d, cryptlen %d\n", err,
2435            req->cryptlen);
2436    #endif /* WOLFKM_DEBUG_AES */
2437
2438    return err;
2439}
2440
2441static struct skcipher_alg xtsAesAlg = {
2442    .base.cra_name          = WOLFKM_AESXTS_NAME,
2443    .base.cra_driver_name   = WOLFKM_AESXTS_DRIVER,
2444    .base.cra_priority      = WOLFSSL_LINUXKM_LKCAPI_PRIORITY,
2445    .base.cra_blocksize     = WC_AES_BLOCK_SIZE,
2446    .base.cra_ctxsize       = sizeof(struct km_AesXtsCtx),
2447    .base.cra_module        = THIS_MODULE,
2448
2449    .min_keysize            = 2 * AES_128_KEY_SIZE,
2450    .max_keysize            = 2 * AES_256_KEY_SIZE,
2451    .ivsize                 = WC_AES_BLOCK_SIZE,
2452    .walksize               = 2 * WC_AES_BLOCK_SIZE,
2453    .init                   = km_AesXtsInit,
2454    .exit                   = km_AesXtsExit,
2455    .setkey                 = km_AesXtsSetKey,
2456    .encrypt                = km_AesXtsEncrypt,
2457    .decrypt                = km_AesXtsDecrypt
2458};
2459static int xtsAesAlg_loaded = 0;
2460
2461#endif /* LINUXKM_LKCAPI_REGISTER_AESXTS */
2462
2463#ifdef LINUXKM_LKCAPI_REGISTER_AESCTR
2464
2465static int km_AesCtrInit(struct crypto_skcipher *tfm)
2466{
2467    struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm);
2468    return km_AesInitCommon(ctx, WOLFKM_AESCTR_DRIVER, 0);
2469}
2470
2471static int km_AesCtrSetKey(struct crypto_skcipher *tfm, const u8 *in_key,
2472                          unsigned int key_len)
2473{
2474    struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm);
2475    return km_AesSetKeyCommon(ctx, in_key, key_len, WOLFKM_AESCTR_DRIVER);
2476}
2477
2478static int km_AesCtrEncrypt(struct skcipher_request *req)
2479{
2480    struct crypto_skcipher * tfm = NULL;
2481    struct km_AesCtx *       ctx = NULL;
2482    struct skcipher_walk     walk;
2483    int                      err;
2484    Aes                      *aes_copy = NULL;
2485
2486    tfm = crypto_skcipher_reqtfm(req);
2487    ctx = crypto_skcipher_ctx(tfm);
2488
2489    err = skcipher_walk_virt(&walk, req, false);
2490
2491    if (unlikely(err)) {
2492        pr_err("%s: skcipher_walk_virt failed: %d\n",
2493               crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2494        return err;
2495    }
2496
2497    /* Copy the cipher state to mitigate races on Aes.reg and Aes.tmp. */
2498    aes_copy = (struct Aes *)malloc(sizeof(Aes));
2499    if (aes_copy == NULL) {
2500        err = -ENOMEM;
2501        goto out;
2502    }
2503    XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes));
2504
2505    err = wc_AesSetIV(aes_copy, walk.iv);
2506
2507    if (unlikely(err)) {
2508        pr_err("%s: wc_AesSetIV failed: %d\n",
2509               crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2510        err = -EINVAL;
2511        goto out;
2512    }
2513
2514    while (walk.nbytes != 0) {
2515        err = wc_AesCtrEncrypt(aes_copy, walk.dst.virt.addr,
2516                               walk.src.virt.addr, walk.nbytes);
2517
2518        if (unlikely(err)) {
2519            pr_err("%s: wc_AesCtrEncrypt failed %d\n",
2520                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2521            err = -EINVAL;
2522            goto out;
2523        }
2524
2525        err = skcipher_walk_done(&walk, 0);
2526
2527        if (unlikely(err)) {
2528            pr_err("%s: skcipher_walk_done failed: %d\n",
2529                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2530            return err;
2531        }
2532    }
2533
2534    /* copy iv from wolfCrypt back to walk.iv */
2535    XMEMCPY(walk.iv, aes_copy->reg, WC_AES_BLOCK_SIZE);
2536
2537out:
2538
2539    if (err && walk.nbytes)
2540        (void)skcipher_walk_done(&walk, err);
2541
2542    km_AesFree(&aes_copy);
2543
2544    #ifdef WOLFKM_DEBUG_AES
2545    pr_info("info: exiting km_AesCtrEncrypt: err %d, cryptlen %d\n", err,
2546            req->cryptlen);
2547    #endif /* WOLFKM_DEBUG_AES */
2548
2549    return err;
2550}
2551
2552static int km_AesCtrDecrypt(struct skcipher_request *req)
2553{
2554    struct crypto_skcipher * tfm = NULL;
2555    struct km_AesCtx *       ctx = NULL;
2556    struct skcipher_walk     walk;
2557    int                      err;
2558    Aes                      *aes_copy = NULL;
2559
2560    tfm = crypto_skcipher_reqtfm(req);
2561    ctx = crypto_skcipher_ctx(tfm);
2562
2563    err = skcipher_walk_virt(&walk, req, false);
2564
2565    if (unlikely(err)) {
2566        pr_err("%s: skcipher_walk_virt failed: %d\n",
2567               crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2568        return err;
2569    }
2570
2571    /* Copy the cipher state to mitigate races on Aes.reg and Aes.tmp. */
2572    aes_copy = (struct Aes *)malloc(sizeof(Aes));
2573    if (aes_copy == NULL) {
2574        err = -ENOMEM;
2575        goto out;
2576    }
2577    XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); /* CTR uses the same
2578                                                       * schedule for encrypt
2579                                                       * and decrypt.
2580                                                       */
2581
2582    err = wc_AesSetIV(aes_copy, walk.iv);
2583
2584    if (unlikely(err)) {
2585        if (! disable_setkey_warnings)
2586            pr_err("%s: wc_AesSetIV failed: %d\n",
2587                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2588        err = -EINVAL;
2589        goto out;
2590    }
2591
2592    while (walk.nbytes != 0) {
2593        /* CTR uses the same function for encrypt and decrypt. */
2594        err = wc_AesCtrEncrypt(aes_copy, walk.dst.virt.addr,
2595                               walk.src.virt.addr, walk.nbytes);
2596
2597        if (unlikely(err)) {
2598            pr_err("%s: wc_AesCtrDecrypt failed: %d\n",
2599                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2600            err = -EINVAL;
2601            goto out;
2602        }
2603
2604        err = skcipher_walk_done(&walk, 0);
2605
2606        if (unlikely(err)) {
2607            pr_err("%s: skcipher_walk_done failed: %d\n",
2608                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2609            goto out;
2610        }
2611    }
2612
2613    /* copy iv from wolfCrypt back to walk.iv */
2614    XMEMCPY(walk.iv, aes_copy->reg, WC_AES_BLOCK_SIZE);
2615
2616out:
2617
2618    if (err && walk.nbytes)
2619        (void)skcipher_walk_done(&walk, err);
2620
2621    km_AesFree(&aes_copy);
2622
2623    #ifdef WOLFKM_DEBUG_AES
2624    pr_info("info: exiting km_AesCtrDecrypt: err %d, cryptlen %d\n", err,
2625            req->cryptlen);
2626    #endif /* WOLFKM_DEBUG_AES */
2627
2628    return err;
2629}
2630
2631static struct skcipher_alg ctrAesAlg = {
2632    .base.cra_name        = WOLFKM_AESCTR_NAME,
2633    .base.cra_driver_name = WOLFKM_AESCTR_DRIVER,
2634    .base.cra_priority    = WOLFSSL_LINUXKM_LKCAPI_PRIORITY,
2635    .base.cra_blocksize   = 1,
2636    .base.cra_ctxsize     = sizeof(struct km_AesCtx),
2637    .base.cra_module      = THIS_MODULE,
2638    .init                 = km_AesCtrInit,
2639    .exit                 = km_AesExit,
2640    .min_keysize          = AES_128_KEY_SIZE,
2641    .max_keysize          = AES_256_KEY_SIZE,
2642    .ivsize               = WC_AES_BLOCK_SIZE,
2643    .setkey               = km_AesCtrSetKey,
2644    .encrypt              = km_AesCtrEncrypt,
2645    .decrypt              = km_AesCtrDecrypt,
2646};
2647static int ctrAesAlg_loaded = 0;
2648
2649#endif /* LINUXKM_LKCAPI_REGISTER_AESCTR */
2650
2651#ifdef LINUXKM_LKCAPI_REGISTER_AESOFB
2652
2653static int km_AesOfbInit(struct crypto_skcipher *tfm)
2654{
2655    struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm);
2656    return km_AesInitCommon(ctx, WOLFKM_AESOFB_DRIVER, 0);
2657}
2658
2659static int km_AesOfbSetKey(struct crypto_skcipher *tfm, const u8 *in_key,
2660                          unsigned int key_len)
2661{
2662    struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm);
2663    return km_AesSetKeyCommon(ctx, in_key, key_len, WOLFKM_AESOFB_DRIVER);
2664}
2665
2666static int km_AesOfbEncrypt(struct skcipher_request *req)
2667{
2668    struct crypto_skcipher * tfm = NULL;
2669    struct km_AesCtx *       ctx = NULL;
2670    struct skcipher_walk     walk;
2671    int                      err;
2672    Aes                      *aes_copy = NULL;
2673
2674    tfm = crypto_skcipher_reqtfm(req);
2675    ctx = crypto_skcipher_ctx(tfm);
2676
2677    err = skcipher_walk_virt(&walk, req, false);
2678
2679    if (unlikely(err)) {
2680        pr_err("%s: skcipher_walk_virt failed: %d\n",
2681               crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2682        return err;
2683    }
2684
2685    /* Copy the cipher state to mitigate races on Aes.reg and Aes.tmp. */
2686    aes_copy = (struct Aes *)malloc(sizeof(Aes));
2687    if (aes_copy == NULL) {
2688        err = -ENOMEM;
2689        goto out;
2690    }
2691    XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes));
2692
2693    err = wc_AesSetIV(aes_copy, walk.iv);
2694
2695    if (unlikely(err)) {
2696        pr_err("%s: wc_AesSetIV failed: %d\n",
2697               crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2698        err = -EINVAL;
2699        goto out;
2700    }
2701
2702    while (walk.nbytes != 0) {
2703        err = wc_AesOfbEncrypt(aes_copy, walk.dst.virt.addr,
2704                               walk.src.virt.addr, walk.nbytes);
2705
2706        if (unlikely(err)) {
2707            pr_err("%s: wc_AesOfbEncrypt failed %d\n",
2708                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2709            err = -EINVAL;
2710            goto out;
2711        }
2712
2713        err = skcipher_walk_done(&walk, 0);
2714
2715        if (unlikely(err)) {
2716            pr_err("%s: skcipher_walk_done failed: %d\n",
2717                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2718            return err;
2719        }
2720    }
2721
2722    /* copy iv from wolfCrypt back to walk.iv */
2723    XMEMCPY(walk.iv, aes_copy->reg, WC_AES_BLOCK_SIZE);
2724
2725out:
2726
2727    if (err && walk.nbytes)
2728        (void)skcipher_walk_done(&walk, err);
2729
2730    km_AesFree(&aes_copy);
2731
2732    #ifdef WOLFKM_DEBUG_AES
2733    pr_info("info: exiting km_AesOfbEncrypt: err %d, cryptlen %d\n", err,
2734            req->cryptlen);
2735    #endif /* WOLFKM_DEBUG_AES */
2736
2737    return err;
2738}
2739
2740static int km_AesOfbDecrypt(struct skcipher_request *req)
2741{
2742    struct crypto_skcipher * tfm = NULL;
2743    struct km_AesCtx *       ctx = NULL;
2744    struct skcipher_walk     walk;
2745    int                      err;
2746    Aes                      *aes_copy = NULL;
2747
2748    tfm = crypto_skcipher_reqtfm(req);
2749    ctx = crypto_skcipher_ctx(tfm);
2750
2751    err = skcipher_walk_virt(&walk, req, false);
2752
2753    if (unlikely(err)) {
2754        pr_err("%s: skcipher_walk_virt failed: %d\n",
2755               crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2756        return err;
2757    }
2758
2759    /* Copy the cipher state to mitigate races on Aes.reg and Aes.tmp. */
2760    aes_copy = (struct Aes *)malloc(sizeof(Aes));
2761    if (aes_copy == NULL) {
2762        err = -ENOMEM;
2763        goto out;
2764    }
2765    XMEMCPY(aes_copy, ctx->aes_encrypt, sizeof(Aes)); /* OFB uses the same
2766                                                       * schedule for encrypt
2767                                                       * and decrypt.
2768                                                       */
2769
2770    err = wc_AesSetIV(aes_copy, walk.iv);
2771
2772    if (unlikely(err)) {
2773        if (! disable_setkey_warnings)
2774            pr_err("%s: wc_AesSetIV failed: %d\n",
2775                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2776        err = -EINVAL;
2777        goto out;
2778    }
2779
2780    while (walk.nbytes != 0) {
2781        err = wc_AesOfbDecrypt(aes_copy, walk.dst.virt.addr,
2782                               walk.src.virt.addr, walk.nbytes);
2783
2784        if (unlikely(err)) {
2785            pr_err("%s: wc_AesOfbDecrypt failed: %d\n",
2786                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2787            err = -EINVAL;
2788            goto out;
2789        }
2790
2791        err = skcipher_walk_done(&walk, 0);
2792
2793        if (unlikely(err)) {
2794            pr_err("%s: skcipher_walk_done failed: %d\n",
2795                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), err);
2796            goto out;
2797        }
2798    }
2799
2800    /* copy iv from wolfCrypt back to walk.iv */
2801    XMEMCPY(walk.iv, aes_copy->reg, WC_AES_BLOCK_SIZE);
2802
2803out:
2804
2805    if (err && walk.nbytes)
2806        (void)skcipher_walk_done(&walk, err);
2807
2808    km_AesFree(&aes_copy);
2809
2810    #ifdef WOLFKM_DEBUG_AES
2811    pr_info("info: exiting km_AesOfbDecrypt: err %d, cryptlen %d\n", err,
2812            req->cryptlen);
2813    #endif /* WOLFKM_DEBUG_AES */
2814
2815    return err;
2816}
2817
2818static struct skcipher_alg ofbAesAlg = {
2819    .base.cra_name        = WOLFKM_AESOFB_NAME,
2820    .base.cra_driver_name = WOLFKM_AESOFB_DRIVER,
2821    .base.cra_priority    = WOLFSSL_LINUXKM_LKCAPI_PRIORITY,
2822    .base.cra_blocksize   = 1,
2823    .base.cra_ctxsize     = sizeof(struct km_AesCtx),
2824    .base.cra_module      = THIS_MODULE,
2825    .init                 = km_AesOfbInit,
2826    .exit                 = km_AesExit,
2827    .min_keysize          = AES_128_KEY_SIZE,
2828    .max_keysize          = AES_256_KEY_SIZE,
2829    .ivsize               = WC_AES_BLOCK_SIZE,
2830    .setkey               = km_AesOfbSetKey,
2831    .encrypt              = km_AesOfbEncrypt,
2832    .decrypt              = km_AesOfbDecrypt,
2833};
2834static int ofbAesAlg_loaded = 0;
2835
2836#endif /* LINUXKM_LKCAPI_REGISTER_AESOFB */
2837
2838#ifdef LINUXKM_LKCAPI_REGISTER_AESECB
2839
2840static int km_AesEcbInit(struct crypto_skcipher *tfm)
2841{
2842    struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm);
2843    return km_AesInitCommon(ctx, WOLFKM_AESECB_DRIVER, 1);
2844}
2845
2846static int km_AesEcbSetKey(struct crypto_skcipher *tfm, const u8 *in_key,
2847                          unsigned int key_len)
2848{
2849    struct km_AesCtx * ctx = crypto_skcipher_ctx(tfm);
2850    return km_AesSetKeyCommon(ctx, in_key, key_len, WOLFKM_AESECB_DRIVER);
2851}
2852
2853static int km_AesEcbEncrypt(struct skcipher_request *req)
2854{
2855    struct crypto_skcipher * tfm = NULL;
2856    struct km_AesCtx *       ctx = NULL;
2857    struct skcipher_walk     walk;
2858    unsigned int             nbytes = 0;
2859    int                      err;
2860    Aes                     *aes;
2861
2862    tfm = crypto_skcipher_reqtfm(req);
2863    ctx = crypto_skcipher_ctx(tfm);
2864
2865    err = skcipher_walk_virt(&walk, req, false);
2866
2867    if (unlikely(err)) {
2868        return err;
2869    }
2870
2871    err = km_AesGet(ctx, 0 /* decrypt_p */, 0 /* copy_p */, &aes);
2872    if (unlikely(err)) {
2873        goto out;
2874    }
2875
2876    while ((nbytes = walk.nbytes) != 0) {
2877        err = wc_AesEcbEncrypt(aes, walk.dst.virt.addr, walk.src.virt.addr,
2878                               nbytes & (~(WC_AES_BLOCK_SIZE - 1)));
2879
2880        if (unlikely(err)) {
2881            pr_err("%s: wc_AesEcbEncrypt failed for %u bytes: %d\n",
2882                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), nbytes, err);
2883            err = -EINVAL;
2884            goto out;
2885        }
2886
2887        nbytes &= WC_AES_BLOCK_SIZE - 1;
2888        err = skcipher_walk_done(&walk, nbytes);
2889    }
2890
2891out:
2892
2893    if (err && walk.nbytes)
2894        (void)skcipher_walk_done(&walk, err);
2895
2896    #ifdef WOLFKM_DEBUG_AES
2897    pr_info("info: exiting km_AesEcbEncrypt: err %d, cryptlen %d\n", err,
2898            req->cryptlen);
2899    #endif /* WOLFKM_DEBUG_AES */
2900
2901    return err;
2902}
2903
2904static int km_AesEcbDecrypt(struct skcipher_request *req)
2905{
2906    struct crypto_skcipher * tfm = NULL;
2907    struct km_AesCtx *       ctx = NULL;
2908    struct skcipher_walk     walk;
2909    unsigned int             nbytes = 0;
2910    int                      err;
2911    Aes                     *aes;
2912
2913    tfm = crypto_skcipher_reqtfm(req);
2914    ctx = crypto_skcipher_ctx(tfm);
2915
2916    err = skcipher_walk_virt(&walk, req, false);
2917
2918    if (unlikely(err)) {
2919        return err;
2920    }
2921
2922    err = km_AesGet(ctx, 1 /* decrypt_p */, 0 /* copy_p */, &aes);
2923    if (unlikely(err)) {
2924        goto out;
2925    }
2926
2927    while ((nbytes = walk.nbytes) != 0) {
2928        err = wc_AesEcbDecrypt(aes, walk.dst.virt.addr, walk.src.virt.addr,
2929                               nbytes & (~(WC_AES_BLOCK_SIZE - 1)));
2930
2931        if (unlikely(err)) {
2932            pr_err("%s: wc_AesEcbDecrypt failed for %u bytes: %d\n",
2933                   crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm)), nbytes, err);
2934            err = -EINVAL;
2935            goto out;
2936        }
2937
2938        nbytes &= WC_AES_BLOCK_SIZE - 1;
2939        err = skcipher_walk_done(&walk, nbytes);
2940    }
2941
2942out:
2943
2944    if (err && walk.nbytes)
2945        (void)skcipher_walk_done(&walk, err);
2946
2947    #ifdef WOLFKM_DEBUG_AES
2948    pr_info("info: exiting km_AesEcbDecrypt: err %d, cryptlen %d\n", err,
2949            req->cryptlen);
2950    #endif /* WOLFKM_DEBUG_AES */
2951
2952    return err;
2953}
2954
2955static struct skcipher_alg ecbAesAlg = {
2956    .base.cra_name        = WOLFKM_AESECB_NAME,
2957    .base.cra_driver_name = WOLFKM_AESECB_DRIVER,
2958    .base.cra_priority    = WOLFSSL_LINUXKM_LKCAPI_PRIORITY,
2959    .base.cra_blocksize   = WC_AES_BLOCK_SIZE,
2960    .base.cra_ctxsize     = sizeof(struct km_AesCtx),
2961    .base.cra_module      = THIS_MODULE,
2962    .init                 = km_AesEcbInit,
2963    .exit                 = km_AesExit,
2964    .min_keysize          = AES_128_KEY_SIZE,
2965    .max_keysize          = AES_256_KEY_SIZE,
2966    .ivsize               = 0,
2967    .setkey               = km_AesEcbSetKey,
2968    .encrypt              = km_AesEcbEncrypt,
2969    .decrypt              = km_AesEcbDecrypt,
2970};
2971static int ecbAesAlg_loaded = 0;
2972
2973#endif /* LINUXKM_LKCAPI_REGISTER_AESECB */
2974
2975/* cipher tests, cribbed from test.c, with supplementary LKCAPI tests: */
2976
2977#ifdef LINUXKM_LKCAPI_REGISTER_AESCBC
2978
2979static int linuxkm_test_aescbc(void)
2980{
2981    int    ret = 0;
2982    struct crypto_skcipher *  tfm = NULL;
2983    struct skcipher_request * req = NULL;
2984    struct scatterlist        src, dst;
2985    Aes    *aes;
2986    int    aes_inited = 0;
2987    static const byte key32[] =
2988    {
2989        0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
2990        0x38, 0x39, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66,
2991        0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
2992        0x38, 0x39, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66
2993    };
2994    static const byte p_vector[] =
2995    /* Now is the time for all good men w/o trailing 0 */
2996    {
2997        0x4e,0x6f,0x77,0x20,0x69,0x73,0x20,0x74,
2998        0x68,0x65,0x20,0x74,0x69,0x6d,0x65,0x20,
2999        0x66,0x6f,0x72,0x20,0x61,0x6c,0x6c,0x20,
3000        0x67,0x6f,0x6f,0x64,0x20,0x6d,0x65,0x6e
3001    };
3002    static const byte iv[] = "1234567890abcdef";
3003
3004    static const byte c_vector[] =
3005    {
3006        0xd7,0xd6,0x04,0x5b,0x4d,0xc4,0x90,0xdf,
3007        0x4a,0x82,0xed,0x61,0x26,0x4e,0x23,0xb3,
3008        0xe4,0xb5,0x85,0x30,0x29,0x4c,0x9d,0xcf,
3009        0x73,0xc9,0x46,0xd1,0xaa,0xc8,0xcb,0x62
3010    };
3011
3012    byte    iv_copy[sizeof(iv)];
3013    byte    enc[sizeof(p_vector)];
3014    byte    dec[sizeof(p_vector)];
3015    u8 *    enc2 = NULL;
3016    u8 *    dec2 = NULL;
3017
3018    aes = (Aes *)malloc(sizeof(*aes));
3019    if (aes == NULL)
3020        return MEMORY_E;
3021
3022    XMEMSET(enc, 0, sizeof(enc));
3023    XMEMSET(dec, 0, sizeof(enc));
3024
3025    ret = wc_AesInit(aes, NULL, INVALID_DEVID);
3026    if (ret) {
3027        pr_err("wolfcrypt wc_AesInit failed with return code %d.\n", ret);
3028        goto test_cbc_end;
3029    }
3030    aes_inited = 1;
3031
3032    ret = wc_AesSetKey(aes, key32, WC_AES_BLOCK_SIZE * 2, iv, AES_ENCRYPTION);
3033    if (ret) {
3034        pr_err("wolfcrypt wc_AesSetKey failed with return code %d\n", ret);
3035        goto test_cbc_end;
3036    }
3037
3038    ret = wc_AesCbcEncrypt(aes, enc, p_vector, sizeof(p_vector));
3039    if (ret) {
3040        pr_err("wolfcrypt wc_AesCbcEncrypt failed with return code %d\n", ret);
3041        goto test_cbc_end;
3042    }
3043
3044    if (XMEMCMP(enc, c_vector, sizeof(c_vector)) != 0) {
3045        pr_err("wolfcrypt wc_AesCbcEncrypt KAT mismatch\n");
3046        return LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
3047    }
3048
3049    /* Re init for decrypt and set flag. */
3050    wc_AesFree(aes);
3051    aes_inited = 0;
3052
3053    ret = wc_AesInit(aes, NULL, INVALID_DEVID);
3054    if (ret) {
3055        pr_err("wolfcrypt wc_AesInit failed with return code %d.\n", ret);
3056        goto test_cbc_end;
3057    }
3058    aes_inited = 1;
3059
3060    ret = wc_AesSetKey(aes, key32, WC_AES_BLOCK_SIZE * 2, iv, AES_DECRYPTION);
3061    if (ret) {
3062        pr_err("wolfcrypt wc_AesSetKey failed with return code %d.\n", ret);
3063        goto test_cbc_end;
3064    }
3065
3066    ret = wc_AesCbcDecrypt(aes, dec, enc, sizeof(p_vector));
3067    if (ret) {
3068        pr_err("wolfcrypt wc_AesCbcDecrypt failed with return code %d\n", ret);
3069        goto test_cbc_end;
3070    }
3071
3072    ret = XMEMCMP(p_vector, dec, sizeof(p_vector));
3073    if (ret) {
3074        pr_err("error: p_vector and dec do not match: %d\n", ret);
3075        goto test_cbc_end;
3076    }
3077
3078    /* now the kernel crypto part */
3079    enc2 = malloc(sizeof(p_vector));
3080    if (!enc2) {
3081        pr_err("error: malloc failed\n");
3082        goto test_cbc_end;
3083    }
3084
3085    dec2 = malloc(sizeof(p_vector));
3086    if (!dec2) {
3087        pr_err("error: malloc failed\n");
3088        goto test_cbc_end;
3089    }
3090
3091    memcpy(dec2, p_vector, sizeof(p_vector));
3092
3093    tfm = crypto_alloc_skcipher(WOLFKM_AESCBC_NAME, 0, 0);
3094    if (IS_ERR(tfm)) {
3095        pr_err("error: allocating AES skcipher algorithm %s failed: %ld\n",
3096               WOLFKM_AESCBC_DRIVER, PTR_ERR(tfm));
3097        tfm = NULL;
3098        goto test_cbc_end;
3099    }
3100
3101#ifndef LINUXKM_LKCAPI_PRIORITY_ALLOW_MASKING
3102    {
3103        const char *driver_name =
3104            crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm));
3105        if (strcmp(driver_name, WOLFKM_AESCBC_DRIVER)) {
3106            pr_err("error: unexpected implementation for %s: %s (expected %s)\n",
3107                   WOLFKM_AESCBC_NAME, driver_name, WOLFKM_AESCBC_DRIVER);
3108            ret = -ENOENT;
3109            goto test_cbc_end;
3110        }
3111    }
3112#endif
3113
3114    ret = crypto_skcipher_setkey(tfm, key32, WC_AES_BLOCK_SIZE * 2);
3115    if (ret) {
3116        pr_err("error: crypto_skcipher_setkey returned: %d\n", ret);
3117        goto test_cbc_end;
3118    }
3119
3120    req = skcipher_request_alloc(tfm, GFP_KERNEL);
3121    if (IS_ERR(req)) {
3122        pr_err("error: allocating AES skcipher request %s failed\n",
3123               WOLFKM_AESCBC_DRIVER);
3124        req = NULL;
3125        goto test_cbc_end;
3126    }
3127
3128    sg_init_one(&src, dec2, sizeof(p_vector));
3129    sg_init_one(&dst, enc2, sizeof(p_vector));
3130
3131    XMEMCPY(iv_copy, iv, sizeof(iv));
3132    skcipher_request_set_crypt(req, &src, &dst, sizeof(p_vector), iv_copy);
3133
3134    ret = crypto_skcipher_encrypt(req);
3135
3136    if (ret) {
3137        pr_err("error: crypto_skcipher_encrypt returned: %d\n", ret);
3138        goto test_cbc_end;
3139    }
3140
3141    ret = XMEMCMP(enc, enc2, sizeof(p_vector));
3142    if (ret) {
3143        pr_err("error: enc and enc2 do not match: %d\n", ret);
3144        goto test_cbc_end;
3145    }
3146
3147    memset(dec2, 0, sizeof(p_vector));
3148    sg_init_one(&src, enc2, sizeof(p_vector));
3149    sg_init_one(&dst, dec2, sizeof(p_vector));
3150
3151    XMEMCPY(iv_copy, iv, sizeof(iv));
3152    skcipher_request_set_crypt(req, &src, &dst, sizeof(p_vector), iv_copy);
3153
3154    ret = crypto_skcipher_decrypt(req);
3155
3156    if (ret) {
3157        pr_err("ERROR: crypto_skcipher_decrypt returned %d\n", ret);
3158        goto test_cbc_end;
3159    }
3160
3161    ret = XMEMCMP(dec, dec2, sizeof(p_vector));
3162    if (ret) {
3163        pr_err("error: dec and dec2 do not match: %d\n", ret);
3164        goto test_cbc_end;
3165    }
3166
3167test_cbc_end:
3168
3169    if (enc2) { free(enc2); }
3170    if (dec2) { free(dec2); }
3171    if (req) { skcipher_request_free(req); }
3172    if (tfm) { crypto_free_skcipher(tfm); }
3173
3174    if (aes_inited)
3175        wc_AesFree(aes);
3176    free(aes);
3177
3178    return ret;
3179}
3180
3181#endif /* LINUXKM_LKCAPI_REGISTER_AESCBC */
3182
3183#ifdef LINUXKM_LKCAPI_REGISTER_AESCFB
3184
3185static int linuxkm_test_aescfb(void)
3186{
3187    int    ret = 0;
3188    struct crypto_skcipher *  tfm = NULL;
3189    struct skcipher_request * req = NULL;
3190    struct scatterlist        src, dst;
3191    Aes    *aes;
3192    int    aes_inited = 0;
3193    static const byte key32[] =
3194    {
3195        0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
3196        0x38, 0x39, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66,
3197        0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
3198        0x38, 0x39, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66
3199    };
3200    static const byte p_vector[] =
3201    /* Now is the time for all good men w/o trailing 0 */
3202    {
3203        0x4e,0x6f,0x77,0x20,0x69,0x73,0x20,0x74,
3204        0x68,0x65,0x20,0x74,0x69,0x6d,0x65,0x20,
3205        0x66,0x6f,0x72,0x20,0x61,0x6c,0x6c,0x20,
3206        0x67,0x6f,0x6f,0x64,0x20,0x6d,0x65,0x6e
3207    };
3208    static const byte iv[] = "1234567890abcdef";
3209    static const byte c_vector[] =
3210    {
3211        0x56,0x35,0x3f,0xdd,0xde,0xa6,0x15,0x87,
3212        0x57,0xdc,0x34,0x62,0x9a,0x68,0x96,0x51,
3213        0xc7,0x09,0xb9,0x4e,0x47,0x6b,0x24,0x72,
3214        0x19,0x5a,0xdf,0x7e,0xba,0xa8,0x01,0xb6
3215    };
3216    byte    iv_copy[sizeof(iv)];
3217    byte    enc[sizeof(p_vector)];
3218    byte    dec[sizeof(p_vector)];
3219    u8 *    enc2 = NULL;
3220    u8 *    dec2 = NULL;
3221
3222    aes = (Aes *)malloc(sizeof(*aes));
3223    if (aes == NULL)
3224        return MEMORY_E;
3225
3226    ret = aes_cfb_test();
3227    if (ret) {
3228        wc_test_render_error_message("aes_cfb_test failed: ", ret);
3229        ret = WC_TEST_RET_DEC_EC(ret);
3230        goto test_cfb_end;
3231    }
3232
3233    XMEMSET(enc, 0, sizeof(enc));
3234    XMEMSET(dec, 0, sizeof(enc));
3235
3236    ret = wc_AesInit(aes, NULL, INVALID_DEVID);
3237    if (ret) {
3238        pr_err("wolfcrypt wc_AesInit failed with return code %d.\n", ret);
3239        goto test_cfb_end;
3240    }
3241    aes_inited = 1;
3242
3243    ret = wc_AesSetKey(aes, key32, WC_AES_BLOCK_SIZE * 2, iv, AES_ENCRYPTION);
3244    if (ret) {
3245        pr_err("wolfcrypt wc_AesSetKey failed with return code %d\n", ret);
3246        goto test_cfb_end;
3247    }
3248
3249    ret = wc_AesCfbEncrypt(aes, enc, p_vector, sizeof(p_vector));
3250    if (ret) {
3251        pr_err("wolfcrypt wc_AesCfbEncrypt failed with return code %d\n", ret);
3252        goto test_cfb_end;
3253    }
3254
3255    if (XMEMCMP(enc, c_vector, sizeof(c_vector)) != 0) {
3256        pr_err("wolfcrypt wc_AesCfbEncrypt KAT mismatch\n");
3257        return LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
3258    }
3259
3260    /* Re init for decrypt and set flag. */
3261    wc_AesFree(aes);
3262    aes_inited = 0;
3263
3264    ret = wc_AesInit(aes, NULL, INVALID_DEVID);
3265    if (ret) {
3266        pr_err("wolfcrypt wc_AesInit failed with return code %d.\n", ret);
3267        goto test_cfb_end;
3268    }
3269    aes_inited = 1;
3270
3271    ret = wc_AesSetKey(aes, key32, WC_AES_BLOCK_SIZE * 2, iv, AES_ENCRYPTION);
3272    if (ret) {
3273        pr_err("wolfcrypt wc_AesSetKey failed with return code %d.\n", ret);
3274        goto test_cfb_end;
3275    }
3276
3277    ret = wc_AesCfbDecrypt(aes, dec, enc, sizeof(p_vector));
3278    if (ret) {
3279        pr_err("wolfcrypt wc_AesCfbDecrypt failed with return code %d\n", ret);
3280        goto test_cfb_end;
3281    }
3282
3283    ret = XMEMCMP(p_vector, dec, sizeof(p_vector));
3284    if (ret) {
3285        pr_err("error: p_vector and dec do not match: %d\n", ret);
3286        goto test_cfb_end;
3287    }
3288
3289    /* now the kernel crypto part */
3290    enc2 = malloc(sizeof(p_vector));
3291    if (!enc2) {
3292        pr_err("error: malloc failed\n");
3293        goto test_cfb_end;
3294    }
3295
3296    dec2 = malloc(sizeof(p_vector));
3297    if (!dec2) {
3298        pr_err("error: malloc failed\n");
3299        goto test_cfb_end;
3300    }
3301
3302    memcpy(dec2, p_vector, sizeof(p_vector));
3303
3304    tfm = crypto_alloc_skcipher(WOLFKM_AESCFB_NAME, 0, 0);
3305    if (IS_ERR(tfm)) {
3306        pr_err("error: allocating AES skcipher algorithm %s failed: %ld\n",
3307               WOLFKM_AESCFB_DRIVER, PTR_ERR(tfm));
3308        tfm = NULL;
3309        goto test_cfb_end;
3310    }
3311
3312    ret = check_skcipher_driver_masking(tfm, WOLFKM_AESCFB_NAME, WOLFKM_AESCFB_DRIVER);
3313    if (ret)
3314        goto test_cfb_end;
3315
3316    ret = crypto_skcipher_setkey(tfm, key32, WC_AES_BLOCK_SIZE * 2);
3317    if (ret) {
3318        pr_err("error: crypto_skcipher_setkey returned: %d\n", ret);
3319        goto test_cfb_end;
3320    }
3321
3322    req = skcipher_request_alloc(tfm, GFP_KERNEL);
3323    if (IS_ERR(req)) {
3324        pr_err("error: allocating AES skcipher request %s failed\n",
3325               WOLFKM_AESCFB_DRIVER);
3326        req = NULL;
3327        goto test_cfb_end;
3328    }
3329
3330    sg_init_one(&src, dec2, sizeof(p_vector));
3331    sg_init_one(&dst, enc2, sizeof(p_vector));
3332
3333    XMEMCPY(iv_copy, iv, sizeof(iv));
3334    skcipher_request_set_crypt(req, &src, &dst, sizeof(p_vector), iv_copy);
3335
3336    ret = crypto_skcipher_encrypt(req);
3337
3338    if (ret) {
3339        pr_err("error: crypto_skcipher_encrypt returned: %d\n", ret);
3340        goto test_cfb_end;
3341    }
3342
3343    ret = XMEMCMP(enc, enc2, sizeof(p_vector));
3344    if (ret) {
3345        pr_err("error: enc and enc2 do not match: %d\n", ret);
3346        goto test_cfb_end;
3347    }
3348
3349    memset(dec2, 0, sizeof(p_vector));
3350    sg_init_one(&src, enc2, sizeof(p_vector));
3351    sg_init_one(&dst, dec2, sizeof(p_vector));
3352
3353    XMEMCPY(iv_copy, iv, sizeof(iv));
3354    skcipher_request_set_crypt(req, &src, &dst, sizeof(p_vector), iv_copy);
3355
3356    ret = crypto_skcipher_decrypt(req);
3357
3358    if (ret) {
3359        pr_err("error: crypto_skcipher_decrypt returned: %d\n", ret);
3360        goto test_cfb_end;
3361    }
3362
3363    ret = XMEMCMP(dec, dec2, sizeof(p_vector));
3364    if (ret) {
3365        pr_err("error: dec and dec2 do not match: %d\n", ret);
3366        goto test_cfb_end;
3367    }
3368
3369test_cfb_end:
3370
3371    if (enc2) { free(enc2); }
3372    if (dec2) { free(dec2); }
3373    if (req) { skcipher_request_free(req); }
3374    if (tfm) { crypto_free_skcipher(tfm); }
3375
3376    if (aes_inited)
3377        wc_AesFree(aes);
3378    free(aes);
3379
3380    return ret;
3381}
3382
3383#endif /* LINUXKM_LKCAPI_REGISTER_AESCFB */
3384
3385#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM
3386
3387static int linuxkm_test_aesgcm(void)
3388{
3389#ifndef WOLFSSL_AESGCM_STREAM
3390    wc_test_ret_t ret = aesgcm_test();
3391    if (ret >= 0)
3392        return check_aead_driver_masking(NULL /* tfm */, WOLFKM_AESGCM_NAME, WOLFKM_AESGCM_DRIVER);
3393    else {
3394        wc_test_render_error_message("aesgcm_test failed: ", ret);
3395        return WC_TEST_RET_DEC_EC(ret);
3396    }
3397#else
3398    int     ret = 0;
3399    struct crypto_aead *  tfm = NULL;
3400    struct aead_request * req = NULL;
3401    struct scatterlist *  src = NULL;
3402    struct scatterlist *  dst = NULL;
3403    Aes    *aes;
3404    int    aes_inited = 0;
3405    static const byte key32[] =
3406    {
3407        0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
3408        0x38, 0x39, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66,
3409        0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37,
3410        0x38, 0x39, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66
3411    };
3412    static const byte p_vector[] =
3413    /* Now is the time for all w/o trailing 0 */
3414    {
3415        0x4e,0x6f,0x77,0x20,0x69,0x73,0x20,0x74,
3416        0x68,0x65,0x20,0x74,0x69,0x6d,0x65,0x20,
3417        0x66,0x6f,0x72,0x20,0x61,0x6c,0x6c,0x20
3418    };
3419    static const byte assoc[] =
3420    {
3421        0xfe, 0xed, 0xfa, 0xce, 0xde, 0xad, 0xbe, 0xef,
3422        0xfe, 0xed, 0xfa, 0xce, 0xde, 0xad, 0xbe, 0xef,
3423        0xab, 0xad, 0xda, 0xd2
3424    };
3425    static const byte ivstr[] = "1234567890ab";
3426    static const byte c_vector[] =
3427    {
3428        0x80,0xb9,0x00,0xdc,0x03,0xb8,0x0e,0xaa,
3429        0x98,0x09,0x75,0x01,0x40,0x09,0xb0,0xc3,
3430        0x7a,0xed,0x2c,0x2e,0x4d,0xe5,0xca,0x80
3431    };
3432    static const byte KAT_authTag[] =
3433    {
3434        0x8d,0xf5,0x76,0xae,0x53,0x20,0x5d,0x9c,
3435        0x01,0x64,0xcd,0xf2,0xec,0x7a,0x13,0x03
3436    };
3437    byte    enc[sizeof(p_vector)];
3438    byte    authTag[WC_AES_BLOCK_SIZE];
3439    byte    dec[sizeof(p_vector)];
3440    u8 *    assoc2 = NULL;
3441    u8 *    enc2 = NULL;
3442    u8 *    dec2 = NULL;
3443    u8 *    iv = NULL;
3444    size_t  encryptLen = sizeof(p_vector);
3445    size_t  decryptLen = sizeof(p_vector) + sizeof(authTag);
3446
3447    /* Init stack variables. */
3448    XMEMSET(enc, 0, sizeof(p_vector));
3449    XMEMSET(dec, 0, sizeof(p_vector));
3450    XMEMSET(authTag, 0, WC_AES_BLOCK_SIZE);
3451
3452    aes = (Aes *)malloc(sizeof(*aes));
3453    if (aes == NULL)
3454        return MEMORY_E;
3455
3456    ret = wc_AesInit(aes, NULL, INVALID_DEVID);
3457    if (ret) {
3458        pr_err("error: wc_AesInit failed with return code %d.\n", ret);
3459        goto test_gcm_end;
3460    }
3461    aes_inited = 1;
3462
3463    ret = wc_AesGcmInit(aes, key32, sizeof(key32)/sizeof(byte), ivstr,
3464                        GCM_NONCE_MID_SZ);
3465    if (ret) {
3466        pr_err("error: wc_AesGcmInit failed with return code %d.\n", ret);
3467        goto test_gcm_end;
3468    }
3469
3470    ret = wc_AesGcmEncryptUpdate(aes, NULL, NULL, 0, assoc, sizeof(assoc));
3471    if (ret) {
3472        pr_err("error: wc_AesGcmEncryptUpdate failed with return code %d\n",
3473               ret);
3474        goto test_gcm_end;
3475    }
3476
3477    ret = wc_AesGcmEncryptUpdate(aes, enc, p_vector, sizeof(p_vector), NULL, 0);
3478    if (ret) {
3479        pr_err("error: wc_AesGcmEncryptUpdate failed with return code %d\n",
3480               ret);
3481        goto test_gcm_end;
3482    }
3483
3484    if (XMEMCMP(enc, c_vector, sizeof(c_vector)) != 0) {
3485        pr_err("wolfcrypt AES-GCM KAT mismatch on ciphertext\n");
3486        ret = LINUXKM_LKCAPI_AESGCM_KAT_MISMATCH_E;
3487        goto test_gcm_end;
3488    }
3489
3490    ret = wc_AesGcmEncryptFinal(aes, authTag, WC_AES_BLOCK_SIZE);
3491    if (ret) {
3492        pr_err("error: wc_AesGcmEncryptFinal failed with return code %d\n",
3493               ret);
3494        goto test_gcm_end;
3495    }
3496
3497    if (XMEMCMP(authTag, KAT_authTag, sizeof(KAT_authTag)) != 0) {
3498        pr_err("wolfcrypt AES-GCM KAT mismatch on authTag\n");
3499        ret = LINUXKM_LKCAPI_AESGCM_KAT_MISMATCH_E;
3500        goto test_gcm_end;
3501    }
3502
3503    ret = wc_AesGcmInit(aes, key32, sizeof(key32)/sizeof(byte), ivstr,
3504                        GCM_NONCE_MID_SZ);
3505    if (ret) {
3506        pr_err("error: wc_AesGcmInit failed with return code %d.\n", ret);
3507        goto test_gcm_end;
3508    }
3509
3510    ret = wc_AesGcmDecryptUpdate(aes, dec, enc, sizeof(p_vector),
3511                                 assoc, sizeof(assoc));
3512    if (ret) {
3513        pr_err("error: wc_AesGcmDecryptUpdate failed with return code %d\n",
3514               ret);
3515        goto test_gcm_end;
3516    }
3517
3518    ret = wc_AesGcmDecryptFinal(aes, authTag, WC_AES_BLOCK_SIZE);
3519    if (ret) {
3520        pr_err("error: wc_AesGcmEncryptFinal failed with return code %d\n",
3521               ret);
3522        goto test_gcm_end;
3523    }
3524
3525    ret = XMEMCMP(p_vector, dec, sizeof(p_vector));
3526    if (ret) {
3527        pr_err("error: gcm: p_vector and dec do not match: %d\n", ret);
3528        goto test_gcm_end;
3529    }
3530
3531    /* now the kernel crypto part */
3532    assoc2 = malloc(sizeof(assoc));
3533    if (! assoc2) {
3534        pr_err("error: malloc failed\n");
3535        goto test_gcm_end;
3536    }
3537    memset(assoc2, 0, sizeof(assoc));
3538    memcpy(assoc2, assoc, sizeof(assoc));
3539
3540    iv = malloc(WC_AES_BLOCK_SIZE);
3541    if (! iv) {
3542        pr_err("error: malloc failed\n");
3543        goto test_gcm_end;
3544    }
3545    memset(iv, 0, WC_AES_BLOCK_SIZE);
3546    memcpy(iv, ivstr, GCM_NONCE_MID_SZ);
3547
3548    enc2 = malloc(decryptLen);
3549    if (! enc2) {
3550        pr_err("error: malloc failed\n");
3551        goto test_gcm_end;
3552    }
3553
3554    dec2 = malloc(decryptLen);
3555    if (! dec2) {
3556        pr_err("error: malloc failed\n");
3557        goto test_gcm_end;
3558    }
3559
3560    memset(enc2, 0, decryptLen);
3561    memset(dec2, 0, decryptLen);
3562    memcpy(dec2, p_vector, sizeof(p_vector));
3563
3564    tfm = crypto_alloc_aead(WOLFKM_AESGCM_NAME, 0, 0);
3565    if (IS_ERR(tfm)) {
3566        pr_err("error: allocating AES skcipher algorithm %s failed: %ld\n",
3567               WOLFKM_AESGCM_DRIVER, PTR_ERR(tfm));
3568        tfm = NULL;
3569        goto test_gcm_end;
3570    }
3571
3572    ret = check_aead_driver_masking(tfm, WOLFKM_AESGCM_NAME, WOLFKM_AESGCM_DRIVER);
3573    if (ret)
3574        goto test_gcm_end;
3575
3576    ret = crypto_aead_setkey(tfm, key32, WC_AES_BLOCK_SIZE * 2);
3577    if (ret) {
3578        pr_err("error: crypto_aead_setkey returned: %d\n", ret);
3579        goto test_gcm_end;
3580    }
3581
3582    ret = crypto_aead_setauthsize(tfm, sizeof(authTag));
3583    if (ret) {
3584        pr_err("error: crypto_aead_setauthsize returned: %d\n", ret);
3585        goto test_gcm_end;
3586    }
3587
3588    req = aead_request_alloc(tfm, GFP_KERNEL);
3589    if (IS_ERR(req)) {
3590        pr_err("error: allocating AES aead request %s failed: %ld\n",
3591               WOLFKM_AESCBC_DRIVER, PTR_ERR(req));
3592        req = NULL;
3593        goto test_gcm_end;
3594    }
3595
3596    src = malloc(sizeof(struct scatterlist) * 2);
3597
3598    if (! src) {
3599        pr_err("error: malloc src failed: %ld\n",
3600               PTR_ERR(src));
3601        goto test_gcm_end;
3602    }
3603
3604    dst = malloc(sizeof(struct scatterlist) * 2);
3605
3606    if (! dst) {
3607        pr_err("error: malloc dst failed: %ld\n",
3608               PTR_ERR(dst));
3609        goto test_gcm_end;
3610    }
3611
3612    sg_init_table(src, 2);
3613    sg_set_buf(src, assoc2, sizeof(assoc));
3614    sg_set_buf(&src[1], dec2, sizeof(p_vector));
3615
3616    sg_init_table(dst, 2);
3617    sg_set_buf(dst, assoc2, sizeof(assoc));
3618    sg_set_buf(&dst[1], enc2, decryptLen);
3619
3620    aead_request_set_callback(req, 0, NULL, NULL);
3621    aead_request_set_ad(req, sizeof(assoc));
3622    aead_request_set_crypt(req, src, dst, sizeof(p_vector), iv);
3623
3624    ret = crypto_aead_encrypt(req);
3625
3626    if (ret) {
3627        pr_err("error: crypto_aead_encrypt returned: %d\n", ret);
3628        goto test_gcm_end;
3629    }
3630
3631    ret = XMEMCMP(enc, enc2, sizeof(p_vector));
3632    if (ret) {
3633        pr_err("error: enc and enc2 do not match: %d\n", ret);
3634        goto test_gcm_end;
3635    }
3636
3637    ret = XMEMCMP(authTag, enc2 + encryptLen, sizeof(authTag));
3638    if (ret) {
3639        pr_err("error: authTags do not match: %d\n", ret);
3640        goto test_gcm_end;
3641    }
3642
3643    /* Now decrypt crypto request. Reverse src and dst. */
3644    memset(dec2, 0, decryptLen);
3645    aead_request_set_ad(req, sizeof(assoc));
3646    aead_request_set_crypt(req, dst, src, decryptLen, iv);
3647
3648    ret = crypto_aead_decrypt(req);
3649
3650    if (ret) {
3651        pr_err("error: crypto_aead_decrypt returned: %d\n", ret);
3652        goto test_gcm_end;
3653    }
3654
3655    ret = XMEMCMP(dec, dec2, sizeof(p_vector));
3656    if (ret) {
3657        pr_err("error: dec and dec2 do not match: %d\n", ret);
3658        goto test_gcm_end;
3659    }
3660
3661test_gcm_end:
3662    if (req) { aead_request_free(req); req = NULL; }
3663    if (tfm) { crypto_free_aead(tfm); tfm = NULL; }
3664
3665    if (src) { free(src); src = NULL; }
3666    if (dst) { free(dst); dst = NULL; }
3667
3668    if (dec2) { free(dec2); dec2 = NULL; }
3669    if (enc2) { free(enc2); enc2 = NULL; }
3670
3671    if (assoc2) { free(assoc2); assoc2 = NULL; }
3672    if (iv) { free(iv); iv = NULL; }
3673
3674    if (aes_inited)
3675        wc_AesFree(aes);
3676    free(aes);
3677
3678    return ret;
3679#endif /* WOLFSSL_AESGCM_STREAM */
3680}
3681
3682#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM */
3683
3684#ifdef LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106
3685
3686static int linuxkm_test_aesgcm_rfc4106(void)
3687{
3688    wc_test_ret_t ret = aesgcm_test();
3689    if (ret >= 0)
3690        return check_aead_driver_masking(NULL /* tfm */, WOLFKM_AESGCM_RFC4106_NAME, WOLFKM_AESGCM_RFC4106_DRIVER);
3691    else {
3692        wc_test_render_error_message("aesgcm_test failed: ", ret);
3693        return WC_TEST_RET_DEC_EC(ret);
3694    }
3695}
3696
3697#endif /* LINUXKM_LKCAPI_REGISTER_AESGCM_RFC4106 */
3698
3699#if defined(LINUXKM_LKCAPI_REGISTER_AESCCM) || defined(LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309)
3700
3701static int aesccm_test_once(void) {
3702    static int once = 0;
3703    static int ret;
3704    if (! once) {
3705        ret = aesccm_test();
3706        once = 1;
3707    }
3708    return ret;
3709}
3710
3711#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM
3712
3713static int  linuxkm_test_aesccm(void) {
3714    return aesccm_test_once();
3715}
3716
3717#endif /* LINUXKM_LKCAPI_REGISTER_AESCCM */
3718
3719#ifdef LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309
3720
3721static int  linuxkm_test_aesccm_rfc4309(void) {
3722    return aesccm_test_once();
3723}
3724
3725#endif /* LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309 */
3726
3727#endif /* LINUXKM_LKCAPI_REGISTER_AESCCM || LINUXKM_LKCAPI_REGISTER_AESCCM_RFC4309 */
3728
3729#ifdef LINUXKM_LKCAPI_REGISTER_AESXTS
3730
3731/* test vectors from
3732 * http://csrc.nist.gov/groups/STM/cavp/block-cipher-modes.html
3733 */
3734#ifdef WOLFSSL_AES_128
3735static int aes_xts_128_test(void)
3736{
3737    XtsAes *aes = NULL;
3738    int aes_inited = 0;
3739    int ret = 0;
3740#define AES_XTS_128_TEST_BUF_SIZ (WC_AES_BLOCK_SIZE * 2 + 8)
3741    unsigned char *buf = NULL;
3742    unsigned char *cipher = NULL;
3743    u8 *    enc2 = NULL;
3744    u8 *    dec2 = NULL;
3745    struct scatterlist *  src = NULL;
3746    struct scatterlist *  dst = NULL;
3747    struct crypto_skcipher *tfm = NULL;
3748    struct skcipher_request *req = NULL;
3749    struct XtsAesStreamData stream;
3750    byte* large_input = NULL;
3751
3752    /* 128 key tests */
3753    static const unsigned char k1[] = {
3754        0xa1, 0xb9, 0x0c, 0xba, 0x3f, 0x06, 0xac, 0x35,
3755        0x3b, 0x2c, 0x34, 0x38, 0x76, 0x08, 0x17, 0x62,
3756        0x09, 0x09, 0x23, 0x02, 0x6e, 0x91, 0x77, 0x18,
3757        0x15, 0xf2, 0x9d, 0xab, 0x01, 0x93, 0x2f, 0x2f
3758    };
3759
3760    static const unsigned char i1[] = {
3761        0x4f, 0xae, 0xf7, 0x11, 0x7c, 0xda, 0x59, 0xc6,
3762        0x6e, 0x4b, 0x92, 0x01, 0x3e, 0x76, 0x8a, 0xd5
3763    };
3764
3765    static const unsigned char p1[] = {
3766        0xeb, 0xab, 0xce, 0x95, 0xb1, 0x4d, 0x3c, 0x8d,
3767        0x6f, 0xb3, 0x50, 0x39, 0x07, 0x90, 0x31, 0x1c
3768    };
3769
3770    /* plain text test of partial block is not from NIST test vector list */
3771    static const unsigned char pp[] = {
3772        0xeb, 0xab, 0xce, 0x95, 0xb1, 0x4d, 0x3c, 0x8d,
3773        0x6f, 0xb3, 0x50, 0x39, 0x07, 0x90, 0x31, 0x1c,
3774        0x6e, 0x4b, 0x92, 0x01, 0x3e, 0x76, 0x8a, 0xd5
3775    };
3776
3777    static const unsigned char c1[] = {
3778        0x77, 0x8a, 0xe8, 0xb4, 0x3c, 0xb9, 0x8d, 0x5a,
3779        0x82, 0x50, 0x81, 0xd5, 0xbe, 0x47, 0x1c, 0x63
3780    };
3781
3782    /* plain text test of partial block is not from NIST test vector list */
3783    static const unsigned char cp[] = {
3784        0x2b, 0xf7, 0x2c, 0xf3, 0xeb, 0x85, 0xef, 0x7b,
3785        0x0b, 0x76, 0xa0, 0xaa, 0xf3, 0x3f, 0x25, 0x8b,
3786        0x77, 0x8a, 0xe8, 0xb4, 0x3c, 0xb9, 0x8d, 0x5a
3787    };
3788
3789    static const unsigned char k2[] = {
3790        0x39, 0x25, 0x79, 0x05, 0xdf, 0xcc, 0x77, 0x76,
3791        0x6c, 0x87, 0x0a, 0x80, 0x6a, 0x60, 0xe3, 0xc0,
3792        0x93, 0xd1, 0x2a, 0xcf, 0xcb, 0x51, 0x42, 0xfa,
3793        0x09, 0x69, 0x89, 0x62, 0x5b, 0x60, 0xdb, 0x16
3794    };
3795
3796    static const unsigned char i2[] = {
3797        0x5c, 0xf7, 0x9d, 0xb6, 0xc5, 0xcd, 0x99, 0x1a,
3798        0x1c, 0x78, 0x81, 0x42, 0x24, 0x95, 0x1e, 0x84
3799    };
3800
3801    static const unsigned char p2[] = {
3802        0xbd, 0xc5, 0x46, 0x8f, 0xbc, 0x8d, 0x50, 0xa1,
3803        0x0d, 0x1c, 0x85, 0x7f, 0x79, 0x1c, 0x5c, 0xba,
3804        0xb3, 0x81, 0x0d, 0x0d, 0x73, 0xcf, 0x8f, 0x20,
3805        0x46, 0xb1, 0xd1, 0x9e, 0x7d, 0x5d, 0x8a, 0x56
3806    };
3807
3808    static const unsigned char c2[] = {
3809        0xd6, 0xbe, 0x04, 0x6d, 0x41, 0xf2, 0x3b, 0x5e,
3810        0xd7, 0x0b, 0x6b, 0x3d, 0x5c, 0x8e, 0x66, 0x23,
3811        0x2b, 0xe6, 0xb8, 0x07, 0xd4, 0xdc, 0xc6, 0x0e,
3812        0xff, 0x8d, 0xbc, 0x1d, 0x9f, 0x7f, 0xc8, 0x22
3813    };
3814
3815#ifndef HAVE_FIPS /* FIPS requires different keys for main and tweak. */
3816    static const unsigned char k3[] = {
3817        0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
3818        0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
3819        0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
3820        0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
3821    };
3822    static const unsigned char i3[] = {
3823        0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
3824        0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
3825    };
3826    static const unsigned char p3[] = {
3827        0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
3828        0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
3829        0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
3830        0x20, 0xff, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
3831        0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20
3832    };
3833    static const unsigned char c3[] = {
3834        0xA2, 0x07, 0x47, 0x76, 0x3F, 0xEC, 0x0C, 0x23,
3835        0x1B, 0xD0, 0xBD, 0x46, 0x9A, 0x27, 0x38, 0x12,
3836        0x95, 0x02, 0x3D, 0x5D, 0xC6, 0x94, 0x51, 0x36,
3837        0xA0, 0x85, 0xD2, 0x69, 0x6E, 0x87, 0x0A, 0xBF,
3838        0xB5, 0x5A, 0xDD, 0xCB, 0x80, 0xE0, 0xFC, 0xCD
3839    };
3840#endif /* HAVE_FIPS */
3841
3842    if ((aes = (XtsAes *)XMALLOC(sizeof(*aes), NULL, DYNAMIC_TYPE_AES))
3843        == NULL)
3844    {
3845        ret = MEMORY_E;
3846        goto out;
3847    }
3848
3849    if ((buf = (unsigned char *)XMALLOC(AES_XTS_128_TEST_BUF_SIZ, NULL,
3850                                        DYNAMIC_TYPE_AES)) == NULL)
3851    {
3852        ret = MEMORY_E;
3853        goto out;
3854    }
3855    if ((cipher = (unsigned char *)XMALLOC(AES_XTS_128_TEST_BUF_SIZ, NULL,
3856                                           DYNAMIC_TYPE_AES)) == NULL)
3857    {
3858        ret = MEMORY_E;
3859        goto out;
3860    }
3861
3862    XMEMSET(buf, 0, AES_XTS_128_TEST_BUF_SIZ);
3863    ret = wc_AesXtsInit(aes, NULL, INVALID_DEVID);
3864    if (ret != 0)
3865        goto out;
3866    else
3867        aes_inited = 1;
3868
3869    ret = wc_AesXtsSetKeyNoInit(aes, k2, sizeof(k2), AES_ENCRYPTION);
3870    if (ret != 0)
3871        goto out;
3872
3873    ret = wc_AesXtsEncrypt(aes, buf, p2, sizeof(p2), i2, sizeof(i2));
3874    if (ret != 0)
3875        goto out;
3876    if (XMEMCMP(c2, buf, sizeof(c2))) {
3877        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
3878        goto out;
3879    }
3880
3881#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && defined(WC_C_DYNAMIC_FALLBACK)
3882    WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(WC_NO_ERR_TRACE(SYSLIB_FAILED_E));
3883    ret = wc_AesXtsEncrypt(aes, buf, p2, sizeof(p2), i2, sizeof(i2));
3884    WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0);
3885    if (ret != 0)
3886        goto out;
3887    if (XMEMCMP(c2, buf, sizeof(c2))) {
3888        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
3889        goto out;
3890    }
3891#endif
3892
3893    XMEMSET(buf, 0, AES_XTS_128_TEST_BUF_SIZ);
3894
3895    ret = wc_AesXtsEncryptInit(aes, i2, sizeof(i2), &stream);
3896    if (ret != 0)
3897        goto out;
3898    ret = wc_AesXtsEncryptUpdate(aes, buf, p2, WC_AES_BLOCK_SIZE, &stream);
3899    if (ret != 0)
3900        goto out;
3901    ret = wc_AesXtsEncryptFinal(aes, buf + WC_AES_BLOCK_SIZE,
3902                                 p2 + WC_AES_BLOCK_SIZE,
3903                                 sizeof(p2) - WC_AES_BLOCK_SIZE, &stream);
3904    if (ret != 0)
3905        goto out;
3906    if (XMEMCMP(c2, buf, sizeof(c2))) {
3907        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
3908        goto out;
3909    }
3910
3911    XMEMSET(buf, 0, AES_XTS_128_TEST_BUF_SIZ);
3912
3913    ret = wc_AesXtsSetKeyNoInit(aes, k1, sizeof(k1), AES_ENCRYPTION);
3914    if (ret != 0)
3915        goto out;
3916    ret = wc_AesXtsEncrypt(aes, buf, p1, sizeof(p1), i1, sizeof(i1));
3917    if (ret != 0)
3918        goto out;
3919    if (XMEMCMP(c1, buf, WC_AES_BLOCK_SIZE)) {
3920        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
3921        goto out;
3922    }
3923
3924#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && defined(WC_C_DYNAMIC_FALLBACK)
3925    WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(WC_NO_ERR_TRACE(SYSLIB_FAILED_E));
3926    ret = wc_AesXtsEncrypt(aes, buf, p1, sizeof(p1), i1, sizeof(i1));
3927    WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0);
3928    if (ret != 0)
3929        goto out;
3930    if (XMEMCMP(c1, buf, WC_AES_BLOCK_SIZE)) {
3931        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
3932        goto out;
3933    }
3934#endif
3935
3936    /* partial block encryption test */
3937    XMEMSET(cipher, 0, AES_XTS_128_TEST_BUF_SIZ);
3938    ret = wc_AesXtsEncrypt(aes, cipher, pp, sizeof(pp), i1, sizeof(i1));
3939    if (ret != 0)
3940        goto out;
3941    if (XMEMCMP(cp, cipher, sizeof(cp))) {
3942        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
3943        goto out;
3944    }
3945
3946#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && defined(WC_C_DYNAMIC_FALLBACK)
3947    WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(WC_NO_ERR_TRACE(SYSLIB_FAILED_E));
3948    XMEMSET(cipher, 0, AES_XTS_128_TEST_BUF_SIZ);
3949    ret = wc_AesXtsEncrypt(aes, cipher, pp, sizeof(pp), i1, sizeof(i1));
3950    WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0);
3951    if (ret != 0)
3952        goto out;
3953    if (XMEMCMP(cp, cipher, sizeof(cp))) {
3954        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
3955        goto out;
3956    }
3957#endif
3958
3959    /* partial block decrypt test */
3960    XMEMSET(buf, 0, AES_XTS_128_TEST_BUF_SIZ);
3961    ret = wc_AesXtsSetKeyNoInit(aes, k1, sizeof(k1), AES_DECRYPTION);
3962    if (ret != 0)
3963        goto out;
3964    ret = wc_AesXtsDecrypt(aes, buf, cipher, sizeof(pp), i1, sizeof(i1));
3965    if (ret != 0)
3966        goto out;
3967    if (XMEMCMP(pp, buf, sizeof(pp))) {
3968        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
3969        goto out;
3970    }
3971
3972#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && defined(WC_C_DYNAMIC_FALLBACK)
3973    WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(WC_NO_ERR_TRACE(SYSLIB_FAILED_E));
3974    XMEMSET(buf, 0, AES_XTS_128_TEST_BUF_SIZ);
3975    ret = wc_AesXtsDecrypt(aes, buf, cipher, sizeof(pp), i1, sizeof(i1));
3976    WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0);
3977    if (ret != 0)
3978        goto out;
3979    if (XMEMCMP(pp, buf, sizeof(pp))) {
3980        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
3981        goto out;
3982    }
3983#endif
3984
3985    /* NIST decrypt test vector */
3986    XMEMSET(buf, 0, AES_XTS_128_TEST_BUF_SIZ);
3987    ret = wc_AesXtsDecrypt(aes, buf, c1, sizeof(c1), i1, sizeof(i1));
3988    if (ret != 0)
3989        goto out;
3990    if (XMEMCMP(p1, buf, WC_AES_BLOCK_SIZE)) {
3991        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
3992        goto out;
3993    }
3994
3995#if defined(DEBUG_VECTOR_REGISTER_ACCESS) && defined(WC_C_DYNAMIC_FALLBACK)
3996    WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(WC_NO_ERR_TRACE(SYSLIB_FAILED_E));
3997    XMEMSET(buf, 0, AES_XTS_128_TEST_BUF_SIZ);
3998    ret = wc_AesXtsDecrypt(aes, buf, c1, sizeof(c1), i1, sizeof(i1));
3999    WC_DEBUG_SET_VECTOR_REGISTERS_RETVAL(0);
4000    if (ret != 0)
4001        goto out;
4002    if (XMEMCMP(p1, buf, WC_AES_BLOCK_SIZE)) {
4003        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
4004        goto out;
4005    }
4006#endif
4007
4008    /* fail case with decrypting using wrong key */
4009    XMEMSET(buf, 0, AES_XTS_128_TEST_BUF_SIZ);
4010    ret = wc_AesXtsDecrypt(aes, buf, c2, sizeof(c2), i2, sizeof(i2));
4011    if (ret != 0)
4012        goto out;
4013    if (XMEMCMP(p2, buf, sizeof(p2)) == 0) { /* fail case with wrong key */
4014        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
4015        goto out;
4016    }
4017
4018    /* set correct key and retest */
4019    XMEMSET(buf, 0, AES_XTS_128_TEST_BUF_SIZ);
4020    ret = wc_AesXtsSetKeyNoInit(aes, k2, sizeof(k2), AES_DECRYPTION);
4021    if (ret != 0)
4022        goto out;
4023    ret = wc_AesXtsDecrypt(aes, buf, c2, sizeof(c2), i2, sizeof(i2));
4024    if (ret != 0)
4025        goto out;
4026    if (XMEMCMP(p2, buf, sizeof(p2))) {
4027        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
4028        goto out;
4029    }
4030
4031#ifndef HAVE_FIPS
4032
4033    /* Test ciphertext stealing in-place. */
4034    XMEMCPY(buf, p3, sizeof(p3));
4035    ret = wc_AesXtsSetKeyNoInit(aes, k3, sizeof(k3), AES_ENCRYPTION);
4036    if (ret != 0)
4037        goto out;
4038
4039    ret = wc_AesXtsEncrypt(aes, buf, buf, sizeof(p3), i3, sizeof(i3));
4040    if (ret != 0)
4041        goto out;
4042    if (XMEMCMP(c3, buf, sizeof(c3))) {
4043        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
4044        goto out;
4045    }
4046
4047    ret = wc_AesXtsSetKeyNoInit(aes, k3, sizeof(k3), AES_DECRYPTION);
4048    if (ret != 0)
4049        goto out;
4050    ret = wc_AesXtsDecrypt(aes, buf, buf, sizeof(c3), i3, sizeof(i3));
4051    if (ret != 0)
4052        goto out;
4053    if (XMEMCMP(p3, buf, sizeof(p3))) {
4054        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
4055        goto out;
4056    }
4057
4058#endif /* HAVE_FIPS */
4059
4060    {
4061    #define LARGE_XTS_SZ        1024
4062        int i;
4063        int j;
4064        int k;
4065
4066        large_input = (byte *)XMALLOC(LARGE_XTS_SZ, NULL,
4067            DYNAMIC_TYPE_TMP_BUFFER);
4068        if (large_input == NULL) {
4069            ret = MEMORY_E;
4070            goto out;
4071        }
4072
4073        for (i = 0; i < (int)LARGE_XTS_SZ; i++)
4074            large_input[i] = (byte)i;
4075
4076        /* first, encrypt block by block then decrypt with a one-shot call. */
4077        for (j = 16; j < (int)LARGE_XTS_SZ; j++) {
4078            ret = wc_AesXtsSetKeyNoInit(aes, k1, sizeof(k1), AES_ENCRYPTION);
4079            if (ret != 0)
4080                goto out;
4081            ret = wc_AesXtsEncryptInit(aes, i1, sizeof(i1), &stream);
4082            if (ret != 0)
4083                goto out;
4084            for (k = 0; k < j; k += WC_AES_BLOCK_SIZE) {
4085                if ((j - k) < WC_AES_BLOCK_SIZE*2)
4086                    ret = wc_AesXtsEncryptFinal(aes, large_input + k, large_input + k, j - k, &stream);
4087                else
4088                    ret = wc_AesXtsEncryptUpdate(aes, large_input + k, large_input + k, WC_AES_BLOCK_SIZE, &stream);
4089                if (ret != 0)
4090                    goto out;
4091                if ((j - k) < WC_AES_BLOCK_SIZE*2)
4092                    break;
4093            }
4094            ret = wc_AesXtsSetKeyNoInit(aes, k1, sizeof(k1), AES_DECRYPTION);
4095            if (ret != 0)
4096                goto out;
4097            ret = wc_AesXtsDecrypt(aes, large_input, large_input, j, i1,
4098                sizeof(i1));
4099            if (ret != 0)
4100                goto out;
4101            for (i = 0; i < j; i++) {
4102                if (large_input[i] != (byte)i) {
4103                    ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
4104                    goto out;
4105                }
4106            }
4107        }
4108
4109        /* second, encrypt with a one-shot call then decrypt block by block. */
4110        for (j = 16; j < (int)LARGE_XTS_SZ; j++) {
4111            ret = wc_AesXtsSetKeyNoInit(aes, k1, sizeof(k1), AES_ENCRYPTION);
4112            if (ret != 0)
4113                goto out;
4114            ret = wc_AesXtsEncrypt(aes, large_input, large_input, j, i1,
4115                sizeof(i1));
4116            if (ret != 0)
4117                goto out;
4118            ret = wc_AesXtsSetKeyNoInit(aes, k1, sizeof(k1), AES_DECRYPTION);
4119            if (ret != 0)
4120                goto out;
4121            ret = wc_AesXtsDecryptInit(aes, i1, sizeof(i1), &stream);
4122            if (ret != 0)
4123                goto out;
4124            for (k = 0; k < j; k += WC_AES_BLOCK_SIZE) {
4125                if ((j - k) < WC_AES_BLOCK_SIZE*2)
4126                    ret = wc_AesXtsDecryptFinal(aes, large_input + k, large_input + k, j - k, &stream);
4127                else
4128                    ret = wc_AesXtsDecryptUpdate(aes, large_input + k, large_input + k, WC_AES_BLOCK_SIZE, &stream);
4129                if (ret != 0)
4130                    goto out;
4131                if ((j - k) < WC_AES_BLOCK_SIZE*2)
4132                    break;
4133            }
4134            for (i = 0; i < j; i++) {
4135                if (large_input[i] != (byte)i) {
4136                    ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
4137                    goto out;
4138                }
4139            }
4140        }
4141    }
4142
4143    /* now the kernel crypto part */
4144
4145    enc2 = XMALLOC(sizeof(pp), NULL, DYNAMIC_TYPE_AES);
4146    if (!enc2) {
4147        pr_err("error: malloc failed\n");
4148        ret = -ENOMEM;
4149        goto test_xts_end;
4150    }
4151
4152    dec2 = XMALLOC(sizeof(pp), NULL, DYNAMIC_TYPE_AES);
4153    if (!dec2) {
4154        pr_err("error: malloc failed\n");
4155        ret = -ENOMEM;
4156        goto test_xts_end;
4157    }
4158
4159    src = XMALLOC(sizeof(*src) * 2, NULL, DYNAMIC_TYPE_AES);
4160    if (! src) {
4161        pr_err("error: malloc failed\n");
4162        ret = -ENOMEM;
4163        goto test_xts_end;
4164    }
4165
4166    dst = XMALLOC(sizeof(*dst) * 2, NULL, DYNAMIC_TYPE_AES);
4167    if (! dst) {
4168        pr_err("error: malloc failed\n");
4169        ret = -ENOMEM;
4170        goto test_xts_end;
4171    }
4172
4173    tfm = crypto_alloc_skcipher(WOLFKM_AESXTS_NAME, 0, 0);
4174    if (IS_ERR(tfm)) {
4175        ret = PTR_ERR(tfm);
4176        pr_err("error: allocating AES skcipher algorithm %s failed: %d\n",
4177               WOLFKM_AESXTS_DRIVER, ret);
4178        tfm = NULL;
4179        goto test_xts_end;
4180    }
4181
4182#ifndef LINUXKM_LKCAPI_PRIORITY_ALLOW_MASKING
4183    {
4184        const char *driver_name =
4185            crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm));
4186        if (strcmp(driver_name, WOLFKM_AESXTS_DRIVER)) {
4187            pr_err("error: unexpected implementation for %s: %s (expected %s)\n",
4188                   WOLFKM_AESXTS_NAME, driver_name, WOLFKM_AESXTS_DRIVER);
4189            ret = -ENOENT;
4190            goto test_xts_end;
4191        }
4192    }
4193#endif
4194
4195    ret = crypto_skcipher_ivsize(tfm);
4196    if (ret != sizeof(stream.tweak_block)) {
4197        pr_err("error: AES skcipher algorithm %s crypto_skcipher_ivsize()"
4198               " returned %d but expected %d\n",
4199               WOLFKM_AESXTS_DRIVER, ret, (int)sizeof(stream.tweak_block));
4200        ret = -EINVAL;
4201        goto test_xts_end;
4202    }
4203
4204    ret = crypto_skcipher_setkey(tfm, k1, sizeof(k1));
4205    if (ret) {
4206        pr_err("error: crypto_skcipher_setkey for %s returned: %d\n",
4207               WOLFKM_AESXTS_NAME, ret);
4208        goto test_xts_end;
4209    }
4210
4211    req = skcipher_request_alloc(tfm, GFP_KERNEL);
4212    if (IS_ERR(req)) {
4213        ret = PTR_ERR(req);
4214        pr_err("error: allocating AES skcipher request %s failed: %d\n",
4215               WOLFKM_AESXTS_DRIVER, ret);
4216        req = NULL;
4217        goto test_xts_end;
4218    }
4219
4220    memcpy(dec2, p1, sizeof(p1));
4221    memset(enc2, 0, sizeof(p1));
4222
4223    sg_init_one(src, dec2, sizeof(p1));
4224    sg_init_one(dst, enc2, sizeof(p1));
4225
4226    memcpy(stream.tweak_block, i1, sizeof(stream.tweak_block));
4227    skcipher_request_set_crypt(req, src, dst, sizeof(p1), stream.tweak_block);
4228
4229    ret = crypto_skcipher_encrypt(req);
4230
4231    if (ret) {
4232        pr_err("error: crypto_skcipher_encrypt returned: %d\n", ret);
4233        goto test_xts_end;
4234    }
4235
4236    ret = XMEMCMP(c1, enc2, sizeof(c1));
4237    if (ret) {
4238        pr_err("error: c1 and enc2 do not match: %d\n", ret);
4239        ret = -EINVAL;
4240        goto test_xts_end;
4241    }
4242
4243    memset(dec2, 0, sizeof(p1));
4244    sg_init_one(src, enc2, sizeof(p1));
4245    sg_init_one(dst, dec2, sizeof(p1));
4246
4247    memcpy(stream.tweak_block, i1, sizeof(stream.tweak_block));
4248    skcipher_request_set_crypt(req, src, dst, sizeof(p1), stream.tweak_block);
4249
4250    ret = crypto_skcipher_decrypt(req);
4251
4252    if (ret) {
4253        pr_err("ERROR: crypto_skcipher_decrypt returned %d\n", ret);
4254        goto test_xts_end;
4255    }
4256
4257    ret = XMEMCMP(p1, dec2, sizeof(p1));
4258    if (ret) {
4259        pr_err("error: p1 and dec2 do not match: %d\n", ret);
4260        ret = -EINVAL;
4261        goto test_xts_end;
4262    }
4263
4264    memcpy(dec2, pp, sizeof(pp));
4265    memset(enc2, 0, sizeof(pp));
4266
4267    sg_init_one(src, dec2, sizeof(pp));
4268    sg_init_one(dst, enc2, sizeof(pp));
4269
4270    memcpy(stream.tweak_block, i1, sizeof(stream.tweak_block));
4271    skcipher_request_set_crypt(req, src, dst, sizeof(pp), stream.tweak_block);
4272
4273    ret = crypto_skcipher_encrypt(req);
4274
4275    if (ret) {
4276        pr_err("error: crypto_skcipher_encrypt returned: %d\n", ret);
4277        goto test_xts_end;
4278    }
4279
4280    ret = XMEMCMP(cp, enc2, sizeof(cp));
4281    if (ret) {
4282        pr_err("error: cp and enc2 do not match: %d\n", ret);
4283        ret = -EINVAL;
4284        goto test_xts_end;
4285    }
4286
4287    memset(dec2, 0, sizeof(pp));
4288    sg_init_one(src, enc2, sizeof(pp));
4289    sg_init_one(dst, dec2, sizeof(pp));
4290
4291    memcpy(stream.tweak_block, i1, sizeof(stream.tweak_block));
4292    skcipher_request_set_crypt(req, src, dst, sizeof(pp), stream.tweak_block);
4293
4294    ret = crypto_skcipher_decrypt(req);
4295
4296    if (ret) {
4297        pr_err("ERROR: crypto_skcipher_decrypt returned %d\n", ret);
4298        goto test_xts_end;
4299    }
4300
4301    ret = XMEMCMP(pp, dec2, sizeof(pp));
4302    if (ret) {
4303        pr_err("error: pp and dec2 do not match: %d\n", ret);
4304        ret = -EINVAL;
4305        goto test_xts_end;
4306    }
4307
4308    test_xts_end:
4309
4310    XFREE(enc2, NULL, DYNAMIC_TYPE_AES);
4311    XFREE(dec2, NULL, DYNAMIC_TYPE_AES);
4312    XFREE(src, NULL, DYNAMIC_TYPE_AES);
4313    XFREE(dst, NULL, DYNAMIC_TYPE_AES);
4314    if (req)
4315        skcipher_request_free(req);
4316    if (tfm)
4317        crypto_free_skcipher(tfm);
4318
4319  out:
4320
4321    XFREE(large_input, NULL, DYNAMIC_TYPE_TMP_BUFFER);
4322
4323    if (aes_inited)
4324        wc_AesXtsFree(aes);
4325
4326    XFREE(buf, NULL, DYNAMIC_TYPE_AES);
4327    XFREE(cipher, NULL, DYNAMIC_TYPE_AES);
4328    XFREE(aes, NULL, DYNAMIC_TYPE_AES);
4329
4330#undef AES_XTS_128_TEST_BUF_SIZ
4331
4332    return ret;
4333}
4334#endif /* WOLFSSL_AES_128 */
4335
4336#ifdef WOLFSSL_AES_256
4337static int aes_xts_256_test(void)
4338{
4339    XtsAes *aes = NULL;
4340    int aes_inited = 0;
4341    int ret = 0;
4342#define AES_XTS_256_TEST_BUF_SIZ (WC_AES_BLOCK_SIZE * 3)
4343    unsigned char *buf = NULL;
4344    unsigned char *cipher = NULL;
4345    u8 *    enc2 = NULL;
4346    u8 *    dec2 = NULL;
4347    struct scatterlist *  src = NULL;
4348    struct scatterlist *  dst = NULL;
4349    struct crypto_skcipher *tfm = NULL;
4350    struct skcipher_request *req = NULL;
4351    struct XtsAesStreamData stream;
4352    byte* large_input = NULL;
4353
4354    /* 256 key tests */
4355    static const unsigned char k1[] = {
4356        0x1e, 0xa6, 0x61, 0xc5, 0x8d, 0x94, 0x3a, 0x0e,
4357        0x48, 0x01, 0xe4, 0x2f, 0x4b, 0x09, 0x47, 0x14,
4358        0x9e, 0x7f, 0x9f, 0x8e, 0x3e, 0x68, 0xd0, 0xc7,
4359        0x50, 0x52, 0x10, 0xbd, 0x31, 0x1a, 0x0e, 0x7c,
4360        0xd6, 0xe1, 0x3f, 0xfd, 0xf2, 0x41, 0x8d, 0x8d,
4361        0x19, 0x11, 0xc0, 0x04, 0xcd, 0xa5, 0x8d, 0xa3,
4362        0xd6, 0x19, 0xb7, 0xe2, 0xb9, 0x14, 0x1e, 0x58,
4363        0x31, 0x8e, 0xea, 0x39, 0x2c, 0xf4, 0x1b, 0x08
4364    };
4365
4366    static const unsigned char i1[] = {
4367        0xad, 0xf8, 0xd9, 0x26, 0x27, 0x46, 0x4a, 0xd2,
4368        0xf0, 0x42, 0x8e, 0x84, 0xa9, 0xf8, 0x75, 0x64
4369    };
4370
4371    static const unsigned char p1[] = {
4372        0x2e, 0xed, 0xea, 0x52, 0xcd, 0x82, 0x15, 0xe1,
4373        0xac, 0xc6, 0x47, 0xe8, 0x10, 0xbb, 0xc3, 0x64,
4374        0x2e, 0x87, 0x28, 0x7f, 0x8d, 0x2e, 0x57, 0xe3,
4375        0x6c, 0x0a, 0x24, 0xfb, 0xc1, 0x2a, 0x20, 0x2e
4376    };
4377
4378    static const unsigned char c1[] = {
4379        0xcb, 0xaa, 0xd0, 0xe2, 0xf6, 0xce, 0xa3, 0xf5,
4380        0x0b, 0x37, 0xf9, 0x34, 0xd4, 0x6a, 0x9b, 0x13,
4381        0x0b, 0x9d, 0x54, 0xf0, 0x7e, 0x34, 0xf3, 0x6a,
4382        0xf7, 0x93, 0xe8, 0x6f, 0x73, 0xc6, 0xd7, 0xdb
4383    };
4384
4385    /* plain text test of partial block is not from NIST test vector list */
4386    static const unsigned char pp[] = {
4387        0xeb, 0xab, 0xce, 0x95, 0xb1, 0x4d, 0x3c, 0x8d,
4388        0x6f, 0xb3, 0x50, 0x39, 0x07, 0x90, 0x31, 0x1c,
4389        0x6e, 0x4b, 0x92, 0x01, 0x3e, 0x76, 0x8a, 0xd5
4390    };
4391
4392    static const unsigned char cp[] = {
4393        0x65, 0x5e, 0x1d, 0x37, 0x4a, 0x91, 0xe7, 0x6c,
4394        0x4f, 0x83, 0x92, 0xbc, 0x5a, 0x10, 0x55, 0x27,
4395        0x61, 0x0e, 0x5a, 0xde, 0xca, 0xc5, 0x12, 0xd8
4396    };
4397
4398    static const unsigned char k2[] = {
4399        0xad, 0x50, 0x4b, 0x85, 0xd7, 0x51, 0xbf, 0xba,
4400        0x69, 0x13, 0xb4, 0xcc, 0x79, 0xb6, 0x5a, 0x62,
4401        0xf7, 0xf3, 0x9d, 0x36, 0x0f, 0x35, 0xb5, 0xec,
4402        0x4a, 0x7e, 0x95, 0xbd, 0x9b, 0xa5, 0xf2, 0xec,
4403        0xc1, 0xd7, 0x7e, 0xa3, 0xc3, 0x74, 0xbd, 0x4b,
4404        0x13, 0x1b, 0x07, 0x83, 0x87, 0xdd, 0x55, 0x5a,
4405        0xb5, 0xb0, 0xc7, 0xe5, 0x2d, 0xb5, 0x06, 0x12,
4406        0xd2, 0xb5, 0x3a, 0xcb, 0x47, 0x8a, 0x53, 0xb4
4407    };
4408
4409    static const unsigned char i2[] = {
4410        0xe6, 0x42, 0x19, 0xed, 0xe0, 0xe1, 0xc2, 0xa0,
4411        0x0e, 0xf5, 0x58, 0x6a, 0xc4, 0x9b, 0xeb, 0x6f
4412    };
4413
4414    static const unsigned char p2[] = {
4415        0x24, 0xcb, 0x76, 0x22, 0x55, 0xb5, 0xa8, 0x00,
4416        0xf4, 0x6e, 0x80, 0x60, 0x56, 0x9e, 0x05, 0x53,
4417        0xbc, 0xfe, 0x86, 0x55, 0x3b, 0xca, 0xd5, 0x89,
4418        0xc7, 0x54, 0x1a, 0x73, 0xac, 0xc3, 0x9a, 0xbd,
4419        0x53, 0xc4, 0x07, 0x76, 0xd8, 0xe8, 0x22, 0x61,
4420        0x9e, 0xa9, 0xad, 0x77, 0xa0, 0x13, 0x4c, 0xfc
4421    };
4422
4423    static const unsigned char c2[] = {
4424        0xa3, 0xc6, 0xf3, 0xf3, 0x82, 0x79, 0x5b, 0x10,
4425        0x87, 0xd7, 0x02, 0x50, 0xdb, 0x2c, 0xd3, 0xb1,
4426        0xa1, 0x62, 0xa8, 0xb6, 0xdc, 0x12, 0x60, 0x61,
4427        0xc1, 0x0a, 0x84, 0xa5, 0x85, 0x3f, 0x3a, 0x89,
4428        0xe6, 0x6c, 0xdb, 0xb7, 0x9a, 0xb4, 0x28, 0x9b,
4429        0xc3, 0xea, 0xd8, 0x10, 0xe9, 0xc0, 0xaf, 0x92
4430    };
4431
4432    if ((aes = (XtsAes *)XMALLOC(sizeof(*aes), NULL, DYNAMIC_TYPE_AES))
4433        == NULL)
4434    {
4435        ret = MEMORY_E;
4436        goto out;
4437    }
4438
4439    if ((buf = (unsigned char *)XMALLOC(AES_XTS_256_TEST_BUF_SIZ, NULL,
4440                                        DYNAMIC_TYPE_AES)) == NULL)
4441    {
4442        ret = MEMORY_E;
4443        goto out;
4444    }
4445    if ((cipher = (unsigned char *)XMALLOC(AES_XTS_256_TEST_BUF_SIZ, NULL,
4446                                           DYNAMIC_TYPE_AES)) == NULL)
4447    {
4448        ret = MEMORY_E;
4449        goto out;
4450    }
4451
4452    ret = wc_AesXtsInit(aes, NULL, INVALID_DEVID);
4453    if (ret != 0)
4454        goto out;
4455    else
4456        aes_inited = 1;
4457
4458    XMEMSET(buf, 0, AES_XTS_256_TEST_BUF_SIZ);
4459    ret = wc_AesXtsSetKeyNoInit(aes, k2, sizeof(k2), AES_ENCRYPTION);
4460    if (ret != 0)
4461        goto out;
4462
4463    ret = wc_AesXtsEncrypt(aes, buf, p2, sizeof(p2), i2, sizeof(i2));
4464    if (ret != 0)
4465        goto out;
4466    if (XMEMCMP(c2, buf, sizeof(c2))) {
4467        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
4468        goto out;
4469    }
4470
4471    XMEMSET(buf, 0, AES_XTS_256_TEST_BUF_SIZ);
4472
4473    ret = wc_AesXtsEncryptInit(aes, i2, sizeof(i2), &stream);
4474    if (ret != 0)
4475        goto out;
4476    ret = wc_AesXtsEncryptUpdate(aes, buf, p2, WC_AES_BLOCK_SIZE, &stream);
4477    if (ret != 0)
4478        goto out;
4479    ret = wc_AesXtsEncryptFinal(aes, buf + WC_AES_BLOCK_SIZE,
4480                                 p2 + WC_AES_BLOCK_SIZE,
4481                                 sizeof(p2) - WC_AES_BLOCK_SIZE, &stream);
4482    if (ret != 0)
4483        goto out;
4484    if (XMEMCMP(c2, buf, sizeof(c2))) {
4485        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
4486        goto out;
4487    }
4488
4489    XMEMSET(buf, 0, AES_XTS_256_TEST_BUF_SIZ);
4490    ret = wc_AesXtsSetKeyNoInit(aes, k1, sizeof(k1), AES_ENCRYPTION);
4491    if (ret != 0)
4492        goto out;
4493    ret = wc_AesXtsEncrypt(aes, buf, p1, sizeof(p1), i1, sizeof(i1));
4494    if (ret != 0)
4495        goto out;
4496    if (XMEMCMP(c1, buf, WC_AES_BLOCK_SIZE)) {
4497        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
4498        goto out;
4499    }
4500
4501    /* partial block encryption test */
4502    XMEMSET(cipher, 0, AES_XTS_256_TEST_BUF_SIZ);
4503    ret = wc_AesXtsEncrypt(aes, cipher, pp, sizeof(pp), i1, sizeof(i1));
4504    if (ret != 0)
4505        goto out;
4506
4507    /* partial block decrypt test */
4508    XMEMSET(buf, 0, AES_XTS_256_TEST_BUF_SIZ);
4509    ret = wc_AesXtsSetKeyNoInit(aes, k1, sizeof(k1), AES_DECRYPTION);
4510    if (ret != 0)
4511        goto out;
4512    ret = wc_AesXtsDecrypt(aes, buf, cipher, sizeof(pp), i1, sizeof(i1));
4513    if (ret != 0)
4514        goto out;
4515    if (XMEMCMP(pp, buf, sizeof(pp))) {
4516        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
4517        goto out;
4518    }
4519
4520    /* NIST decrypt test vector */
4521    XMEMSET(buf, 0, AES_XTS_256_TEST_BUF_SIZ);
4522    ret = wc_AesXtsDecrypt(aes, buf, c1, sizeof(c1), i1, sizeof(i1));
4523    if (ret != 0)
4524        goto out;
4525    if (XMEMCMP(p1, buf, WC_AES_BLOCK_SIZE)) {
4526        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
4527        goto out;
4528    }
4529
4530    XMEMSET(buf, 0, AES_XTS_256_TEST_BUF_SIZ);
4531    ret = wc_AesXtsSetKeyNoInit(aes, k2, sizeof(k2), AES_DECRYPTION);
4532    if (ret != 0)
4533        goto out;
4534    ret = wc_AesXtsDecrypt(aes, buf, c2, sizeof(c2), i2, sizeof(i2));
4535    if (ret != 0)
4536        goto out;
4537    if (XMEMCMP(p2, buf, sizeof(p2))) {
4538        ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
4539        goto out;
4540    }
4541
4542    {
4543    #define LARGE_XTS_SZ        1024
4544        int i;
4545        int j;
4546        int k;
4547
4548        large_input = (byte *)XMALLOC(LARGE_XTS_SZ, NULL,
4549            DYNAMIC_TYPE_TMP_BUFFER);
4550        if (large_input == NULL) {
4551            ret = MEMORY_E;
4552            goto out;
4553        }
4554
4555        for (i = 0; i < (int)LARGE_XTS_SZ; i++)
4556            large_input[i] = (byte)i;
4557
4558        /* first, encrypt block by block then decrypt with a one-shot call. */
4559        for (j = 16; j < (int)LARGE_XTS_SZ; j++) {
4560            ret = wc_AesXtsSetKeyNoInit(aes, k1, sizeof(k1), AES_ENCRYPTION);
4561            if (ret != 0)
4562                goto out;
4563            ret = wc_AesXtsEncryptInit(aes, i1, sizeof(i1), &stream);
4564            if (ret != 0)
4565                goto out;
4566            for (k = 0; k < j; k += WC_AES_BLOCK_SIZE) {
4567                if ((j - k) < WC_AES_BLOCK_SIZE*2)
4568                    ret = wc_AesXtsEncryptFinal(aes, large_input + k, large_input + k, j - k, &stream);
4569                else
4570                    ret = wc_AesXtsEncryptUpdate(aes, large_input + k, large_input + k, WC_AES_BLOCK_SIZE, &stream);
4571                if (ret != 0)
4572                    goto out;
4573                if ((j - k) < WC_AES_BLOCK_SIZE*2)
4574                    break;
4575            }
4576            ret = wc_AesXtsSetKeyNoInit(aes, k1, sizeof(k1), AES_DECRYPTION);
4577            if (ret != 0)
4578                goto out;
4579            ret = wc_AesXtsDecrypt(aes, large_input, large_input, j, i1,
4580                sizeof(i1));
4581            if (ret != 0)
4582                goto out;
4583            for (i = 0; i < j; i++) {
4584                if (large_input[i] != (byte)i) {
4585                    ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
4586                    goto out;
4587                }
4588            }
4589        }
4590
4591        /* second, encrypt with a one-shot call then decrypt block by block. */
4592        for (j = 16; j < (int)LARGE_XTS_SZ; j++) {
4593            ret = wc_AesXtsSetKeyNoInit(aes, k1, sizeof(k1), AES_ENCRYPTION);
4594            if (ret != 0)
4595                goto out;
4596            ret = wc_AesXtsEncrypt(aes, large_input, large_input, j, i1,
4597                sizeof(i1));
4598            if (ret != 0)
4599                goto out;
4600            ret = wc_AesXtsSetKeyNoInit(aes, k1, sizeof(k1), AES_DECRYPTION);
4601            if (ret != 0)
4602                goto out;
4603            ret = wc_AesXtsDecryptInit(aes, i1, sizeof(i1), &stream);
4604            if (ret != 0)
4605                goto out;
4606            for (k = 0; k < j; k += WC_AES_BLOCK_SIZE) {
4607                if ((j - k) < WC_AES_BLOCK_SIZE*2)
4608                    ret = wc_AesXtsDecryptFinal(aes, large_input + k, large_input + k, j - k, &stream);
4609                else
4610                    ret = wc_AesXtsDecryptUpdate(aes, large_input + k, large_input + k, WC_AES_BLOCK_SIZE, &stream);
4611                if (ret != 0)
4612                    goto out;
4613                if ((j - k) < WC_AES_BLOCK_SIZE*2)
4614                    break;
4615            }
4616            for (i = 0; i < j; i++) {
4617                if (large_input[i] != (byte)i) {
4618                    ret = LINUXKM_LKCAPI_AES_KAT_MISMATCH_E;
4619                    goto out;
4620                }
4621            }
4622        }
4623    }
4624
4625    /* now the kernel crypto part */
4626
4627    enc2 = XMALLOC(sizeof(p1), NULL, DYNAMIC_TYPE_AES);
4628    if (!enc2) {
4629        pr_err("error: malloc failed\n");
4630        ret = -ENOMEM;
4631        goto test_xts_end;
4632    }
4633
4634    dec2 = XMALLOC(sizeof(p1), NULL, DYNAMIC_TYPE_AES);
4635    if (!dec2) {
4636        pr_err("error: malloc failed\n");
4637        ret = -ENOMEM;
4638        goto test_xts_end;
4639    }
4640
4641    src = XMALLOC(sizeof(*src) * 2, NULL, DYNAMIC_TYPE_AES);
4642    if (! src) {
4643        pr_err("error: malloc failed\n");
4644        ret = -ENOMEM;
4645        goto test_xts_end;
4646    }
4647
4648    dst = XMALLOC(sizeof(*dst) * 2, NULL, DYNAMIC_TYPE_AES);
4649    if (! dst) {
4650        pr_err("error: malloc failed\n");
4651        ret = -ENOMEM;
4652        goto test_xts_end;
4653    }
4654
4655    tfm = crypto_alloc_skcipher(WOLFKM_AESXTS_NAME, 0, 0);
4656    if (IS_ERR(tfm)) {
4657        ret = PTR_ERR(tfm);
4658        pr_err("error: allocating AES skcipher algorithm %s failed: %d\n",
4659               WOLFKM_AESXTS_DRIVER, ret);
4660        tfm = NULL;
4661        goto test_xts_end;
4662    }
4663
4664#ifndef LINUXKM_LKCAPI_PRIORITY_ALLOW_MASKING
4665    {
4666        const char *driver_name = crypto_tfm_alg_driver_name(crypto_skcipher_tfm(tfm));
4667        if (strcmp(driver_name, WOLFKM_AESXTS_DRIVER)) {
4668            pr_err("error: unexpected implementation for %s: %s (expected %s)\n",
4669                   WOLFKM_AESXTS_NAME, driver_name, WOLFKM_AESXTS_DRIVER);
4670            ret = -ENOENT;
4671            goto test_xts_end;
4672        }
4673    }
4674#endif
4675
4676    ret = crypto_skcipher_ivsize(tfm);
4677    if (ret != sizeof(stream.tweak_block)) {
4678        pr_err("error: AES skcipher algorithm %s crypto_skcipher_ivsize()"
4679               " returned %d but expected %d\n",
4680               WOLFKM_AESXTS_DRIVER, ret, (int)sizeof(stream.tweak_block));
4681        ret = -EINVAL;
4682        goto test_xts_end;
4683    }
4684
4685    ret = crypto_skcipher_setkey(tfm, k1, sizeof(k1));
4686    if (ret) {
4687        pr_err("error: crypto_skcipher_setkey for %s returned: %d\n",
4688               WOLFKM_AESXTS_NAME, ret);
4689        goto test_xts_end;
4690    }
4691
4692    req = skcipher_request_alloc(tfm, GFP_KERNEL);
4693    if (IS_ERR(req)) {
4694        ret = PTR_ERR(req);
4695        pr_err("error: allocating AES skcipher request %s failed: %d\n",
4696               WOLFKM_AESXTS_DRIVER, ret);
4697        req = NULL;
4698        goto test_xts_end;
4699    }
4700
4701    memcpy(dec2, p1, sizeof(p1));
4702    memset(enc2, 0, sizeof(p1));
4703
4704    sg_init_one(src, dec2, sizeof(p1));
4705    sg_init_one(dst, enc2, sizeof(p1));
4706
4707    memcpy(stream.tweak_block, i1, sizeof(stream.tweak_block));
4708    skcipher_request_set_crypt(req, src, dst, sizeof(p1), stream.tweak_block);
4709
4710    ret = crypto_skcipher_encrypt(req);
4711
4712    if (ret) {
4713        pr_err("error: crypto_skcipher_encrypt returned: %d\n", ret);
4714        goto test_xts_end;
4715    }
4716
4717    ret = XMEMCMP(c1, enc2, sizeof(c1));
4718    if (ret) {
4719        pr_err("error: c1 and enc2 do not match: %d\n", ret);
4720        ret = -EINVAL;
4721        goto test_xts_end;
4722    }
4723
4724    memset(dec2, 0, sizeof(p1));
4725    sg_init_one(src, enc2, sizeof(p1));
4726    sg_init_one(dst, dec2, sizeof(p1));
4727
4728    memcpy(stream.tweak_block, i1, sizeof(stream.tweak_block));
4729    skcipher_request_set_crypt(req, src, dst, sizeof(p1), stream.tweak_block);
4730
4731    ret = crypto_skcipher_decrypt(req);
4732
4733    if (ret) {
4734        pr_err("ERROR: crypto_skcipher_decrypt returned %d\n", ret);
4735        goto test_xts_end;
4736    }
4737
4738    ret = XMEMCMP(p1, dec2, sizeof(p1));
4739    if (ret) {
4740        pr_err("error: p1 and dec2 do not match: %d\n", ret);
4741        ret = -EINVAL;
4742        goto test_xts_end;
4743    }
4744
4745    memcpy(dec2, pp, sizeof(pp));
4746    memset(enc2, 0, sizeof(pp));
4747
4748    sg_init_one(src, dec2, sizeof(pp));
4749    sg_init_one(dst, enc2, sizeof(pp));
4750
4751    memcpy(stream.tweak_block, i1, sizeof(stream.tweak_block));
4752    skcipher_request_set_crypt(req, src, dst, sizeof(pp), stream.tweak_block);
4753
4754    ret = crypto_skcipher_encrypt(req);
4755
4756    if (ret) {
4757        pr_err("error: crypto_skcipher_encrypt returned: %d\n", ret);
4758        goto test_xts_end;
4759    }
4760
4761    ret = XMEMCMP(cp, enc2, sizeof(cp));
4762    if (ret) {
4763        pr_err("error: cp and enc2 do not match: %d\n", ret);
4764        ret = -EINVAL;
4765        goto test_xts_end;
4766    }
4767
4768    memset(dec2, 0, sizeof(pp));
4769    sg_init_one(src, enc2, sizeof(pp));
4770    sg_init_one(dst, dec2, sizeof(pp));
4771
4772    memcpy(stream.tweak_block, i1, sizeof(stream.tweak_block));
4773    skcipher_request_set_crypt(req, src, dst, sizeof(pp), stream.tweak_block);
4774
4775    ret = crypto_skcipher_decrypt(req);
4776
4777    if (ret) {
4778        pr_err("ERROR: crypto_skcipher_decrypt returned %d\n", ret);
4779        goto test_xts_end;
4780    }
4781
4782    ret = XMEMCMP(pp, dec2, sizeof(pp));
4783    if (ret) {
4784        pr_err("error: pp and dec2 do not match: %d\n", ret);
4785        ret = -EINVAL;
4786        goto test_xts_end;
4787    }
4788
4789    test_xts_end:
4790
4791    XFREE(enc2, NULL, DYNAMIC_TYPE_AES);
4792    XFREE(dec2, NULL, DYNAMIC_TYPE_AES);
4793    XFREE(src, NULL, DYNAMIC_TYPE_AES);
4794    XFREE(dst, NULL, DYNAMIC_TYPE_AES);
4795    if (req)
4796        skcipher_request_free(req);
4797    if (tfm)
4798        crypto_free_skcipher(tfm);
4799
4800  out:
4801
4802    XFREE(large_input, NULL, DYNAMIC_TYPE_TMP_BUFFER);
4803
4804    if (aes_inited)
4805        wc_AesXtsFree(aes);
4806
4807    XFREE(buf, NULL, DYNAMIC_TYPE_AES);
4808    XFREE(cipher, NULL, DYNAMIC_TYPE_AES);
4809
4810    XFREE(aes, NULL, DYNAMIC_TYPE_AES);
4811
4812#undef AES_XTS_256_TEST_BUF_SIZ
4813
4814    return ret;
4815}
4816#endif /* WOLFSSL_AES_256 */
4817
4818static int linuxkm_test_aesxts(void) {
4819    int ret;
4820
4821    #ifdef WOLFSSL_AES_128
4822    ret = aes_xts_128_test();
4823    if (ret != 0) {
4824        pr_err("aes_xts_128_test() failed with retval %d.\n", ret);
4825        goto out;
4826    }
4827    #endif
4828    #ifdef WOLFSSL_AES_256
4829    ret = aes_xts_256_test();
4830    if (ret != 0) {
4831        pr_err("aes_xts_256_test() failed with retval %d.\n", ret);
4832        goto out;
4833    }
4834    #endif
4835
4836out:
4837
4838    return ret;
4839}
4840
4841#endif /* LINUXKM_LKCAPI_REGISTER_AESXTS */
4842
4843#ifdef LINUXKM_LKCAPI_REGISTER_AESCTR
4844
4845static int linuxkm_test_aesctr(void) {
4846    wc_test_ret_t ret = aes_ctr_test();
4847    if (ret >= 0)
4848        return check_skcipher_driver_masking(NULL /* tfm */, WOLFKM_AESCTR_NAME, WOLFKM_AESCTR_DRIVER);
4849    else {
4850        wc_test_render_error_message("aes_ctr_test failed: ", ret);
4851        return WC_TEST_RET_DEC_EC(ret);
4852    }
4853}
4854
4855#endif /* LINUXKM_LKCAPI_REGISTER_AESCTR */
4856
4857#ifdef LINUXKM_LKCAPI_REGISTER_AESOFB
4858
4859static int linuxkm_test_aesofb(void) {
4860    wc_test_ret_t ret = aesofb_test();
4861    if (ret >= 0)
4862        return check_skcipher_driver_masking(NULL /* tfm */, WOLFKM_AESOFB_NAME, WOLFKM_AESOFB_DRIVER);
4863    else {
4864        wc_test_render_error_message("aesofb_test failed: ", ret);
4865        return WC_TEST_RET_DEC_EC(ret);
4866    }
4867}
4868
4869#endif /* LINUXKM_LKCAPI_REGISTER_AESOFB */
4870
4871#ifdef LINUXKM_LKCAPI_REGISTER_AESECB
4872
4873static int linuxkm_test_aesecb(void) {
4874    wc_test_ret_t ret = aes_test();
4875    if (ret >= 0)
4876        return check_skcipher_driver_masking(NULL /* tfm */, WOLFKM_AESECB_NAME, WOLFKM_AESECB_DRIVER);
4877    else {
4878        wc_test_render_error_message("aes_test failed: ", ret);
4879        return WC_TEST_RET_DEC_EC(ret);
4880    }
4881}
4882
4883#endif /* LINUXKM_LKCAPI_REGISTER_AESECB */
4884
4885#endif /* LINUXKM_LKCAPI_REGISTER_AES */
4886
4887#endif /* !WC_SKIP_INCLUDED_C_FILES */