luna - wolfssl/linuxkm/lkcapi_sha

   1/* lkcapi_sha_glue.c -- glue logic for SHA*
   2 *
   3 * Copyright (C) 2006-2026 wolfSSL Inc.
   4 *
   5 * This file is part of wolfSSL.
   6 *
   7 * wolfSSL is free software; you can redistribute it and/or modify
   8 * it under the terms of the GNU General Public License as published by
   9 * the Free Software Foundation; either version 3 of the License, or
  10 * (at your option) any later version.
  11 *
  12 * wolfSSL is distributed in the hope that it will be useful,
  13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 * GNU General Public License for more details.
  16 *
  17 * You should have received a copy of the GNU General Public License
  18 * along with this program; if not, write to the Free Software
  19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
  20 */
  21
  22/* included by linuxkm/lkcapi_glue.c */
  23#ifndef WC_SKIP_INCLUDED_C_FILES
  24
  25#ifndef LINUXKM_LKCAPI_REGISTER
  26    #error lkcapi_sha_glue.c included in non-LINUXKM_LKCAPI_REGISTER project.
  27#endif
  28
  29#if defined(WC_LINUXKM_C_FALLBACK_IN_SHIMS) && defined(USE_INTEL_SPEEDUP)
  30    #error SHA* WC_LINUXKM_C_FALLBACK_IN_SHIMS is not currently supported.
  31#endif
  32
  33#ifdef NO_LINUXKM_DRBG_GET_RANDOM_BYTES
  34    #undef LINUXKM_DRBG_GET_RANDOM_BYTES
  35/* setup for LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT is in linuxkm_wc_port.h */
  36#elif defined(LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT) && \
  37    (defined(WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS) || \
  38     defined(WOLFSSL_LINUXKM_USE_GET_RANDOM_KPROBES))
  39    #ifndef LINUXKM_DRBG_GET_RANDOM_BYTES
  40        #define LINUXKM_DRBG_GET_RANDOM_BYTES
  41    #endif
  42#else
  43    #ifdef LINUXKM_DRBG_GET_RANDOM_BYTES
  44        #error LINUXKM_DRBG_GET_RANDOM_BYTES configured with no callback model configured.
  45        #undef LINUXKM_DRBG_GET_RANDOM_BYTES
  46    #endif
  47#endif
  48
  49#include <wolfssl/wolfcrypt/sha.h>
  50#include <wolfssl/wolfcrypt/hmac.h>
  51
  52#define WOLFKM_SHA1_NAME "sha1"
  53#define WOLFKM_SHA2_224_NAME "sha224"
  54#define WOLFKM_SHA2_256_NAME "sha256"
  55#define WOLFKM_SHA2_384_NAME "sha384"
  56#define WOLFKM_SHA2_512_NAME "sha512"
  57#define WOLFKM_SHA3_224_NAME "sha3-224"
  58#define WOLFKM_SHA3_256_NAME "sha3-256"
  59#define WOLFKM_SHA3_384_NAME "sha3-384"
  60#define WOLFKM_SHA3_512_NAME "sha3-512"
  61
  62#define WOLFKM_SHA1_HMAC_NAME "hmac(sha1)"
  63#define WOLFKM_SHA2_224_HMAC_NAME "hmac(sha224)"
  64#define WOLFKM_SHA2_256_HMAC_NAME "hmac(sha256)"
  65#define WOLFKM_SHA2_384_HMAC_NAME "hmac(sha384)"
  66#define WOLFKM_SHA2_512_HMAC_NAME "hmac(sha512)"
  67#define WOLFKM_SHA3_224_HMAC_NAME "hmac(sha3-224)"
  68#define WOLFKM_SHA3_256_HMAC_NAME "hmac(sha3-256)"
  69#define WOLFKM_SHA3_384_HMAC_NAME "hmac(sha3-384)"
  70#define WOLFKM_SHA3_512_HMAC_NAME "hmac(sha3-512)"
  71
  72#define WOLFKM_STDRNG_NAME "stdrng"
  73
  74#if defined(USE_INTEL_SPEEDUP)
  75    #ifndef NO_AVX2_SUPPORT
  76        #define WOLFKM_SHA_DRIVER_ISA_EXT "-avx2"
  77    #else
  78        #define WOLFKM_SHA_DRIVER_ISA_EXT "-avx"
  79    #endif
  80#else
  81    #define WOLFKM_SHA_DRIVER_ISA_EXT ""
  82#endif
  83
  84#define WOLFKM_SHA_DRIVER_SUFFIX \
  85    WOLFKM_SHA_DRIVER_ISA_EXT WOLFKM_DRIVER_SUFFIX_BASE
  86
  87#define WOLFKM_SHA1_DRIVER ("sha1" WOLFKM_SHA_DRIVER_SUFFIX)
  88#define WOLFKM_SHA2_224_DRIVER ("sha224" WOLFKM_SHA_DRIVER_SUFFIX)
  89#define WOLFKM_SHA2_256_DRIVER ("sha256" WOLFKM_SHA_DRIVER_SUFFIX)
  90#define WOLFKM_SHA2_384_DRIVER ("sha384" WOLFKM_SHA_DRIVER_SUFFIX)
  91#define WOLFKM_SHA2_512_DRIVER ("sha512" WOLFKM_SHA_DRIVER_SUFFIX)
  92#define WOLFKM_SHA3_224_DRIVER ("sha3-224" WOLFKM_SHA_DRIVER_SUFFIX)
  93#define WOLFKM_SHA3_256_DRIVER ("sha3-256" WOLFKM_SHA_DRIVER_SUFFIX)
  94#define WOLFKM_SHA3_384_DRIVER ("sha3-384" WOLFKM_SHA_DRIVER_SUFFIX)
  95#define WOLFKM_SHA3_512_DRIVER ("sha3-512" WOLFKM_SHA_DRIVER_SUFFIX)
  96
  97#define WOLFKM_SHA1_HMAC_DRIVER ("hmac-sha1" WOLFKM_SHA_DRIVER_SUFFIX)
  98#define WOLFKM_SHA2_224_HMAC_DRIVER ("hmac-sha224" WOLFKM_SHA_DRIVER_SUFFIX)
  99#define WOLFKM_SHA2_256_HMAC_DRIVER ("hmac-sha256" WOLFKM_SHA_DRIVER_SUFFIX)
 100#define WOLFKM_SHA2_384_HMAC_DRIVER ("hmac-sha384" WOLFKM_SHA_DRIVER_SUFFIX)
 101#define WOLFKM_SHA2_512_HMAC_DRIVER ("hmac-sha512" WOLFKM_SHA_DRIVER_SUFFIX)
 102#define WOLFKM_SHA3_224_HMAC_DRIVER ("hmac-sha3-224" WOLFKM_SHA_DRIVER_SUFFIX)
 103#define WOLFKM_SHA3_256_HMAC_DRIVER ("hmac-sha3-256" WOLFKM_SHA_DRIVER_SUFFIX)
 104#define WOLFKM_SHA3_384_HMAC_DRIVER ("hmac-sha3-384" WOLFKM_SHA_DRIVER_SUFFIX)
 105#define WOLFKM_SHA3_512_HMAC_DRIVER ("hmac-sha3-512" WOLFKM_SHA_DRIVER_SUFFIX)
 106
 107/* "nopr" signifies no "prediction resistance".  Prediction resistance entails
 108 * implicit reseeding of the DRBG each time its generator method is called,
 109 * which reduces performance and can rapidly lead to temporary entropy
 110 * exhaustion.  A caller that really needs PR can pass in seed data in its call
 111 * to our rng_alg.generate() implementation.
 112 */
 113
 114#ifdef HAVE_ENTROPY_MEMUSE
 115    #define WOLFKM_STDRNG_WOLFENTROPY "-wolfentropy"
 116#else
 117    #define WOLFKM_STDRNG_WOLFENTROPY ""
 118#endif
 119
 120#if defined(HAVE_INTEL_RDSEED) || defined(HAVE_AMD_RDSEED)
 121    #define WOLFKM_STDRNG_RDSEED "-rdseed"
 122#else
 123    #define WOLFKM_STDRNG_RDSEED ""
 124#endif
 125
 126#ifdef LINUXKM_DRBG_GET_RANDOM_BYTES
 127    #define WOLFKM_STDRNG_DRIVER ("sha2-256-drbg-nopr" \
 128                                  WOLFKM_STDRNG_WOLFENTROPY \
 129                                  WOLFKM_STDRNG_RDSEED \
 130                                  WOLFKM_DRIVER_SUFFIX_BASE \
 131                                  "-with-global-replace")
 132#else
 133    #define WOLFKM_STDRNG_DRIVER ("sha2-256-drbg-nopr" \
 134                                  WOLFKM_STDRNG_WOLFENTROPY \
 135                                  WOLFKM_STDRNG_RDSEED \
 136                                  WOLFKM_DRIVER_SUFFIX_BASE)
 137#endif
 138
 139#ifdef LINUXKM_LKCAPI_REGISTER_SHA_ALL
 140    #define LINUXKM_LKCAPI_REGISTER_SHA1
 141    #define LINUXKM_LKCAPI_REGISTER_SHA2
 142    #define LINUXKM_LKCAPI_REGISTER_SHA3
 143#endif
 144
 145#ifdef LINUXKM_LKCAPI_DONT_REGISTER_SHA_ALL
 146    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA1
 147    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA2
 148    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA3
 149#endif
 150
 151#ifdef LINUXKM_LKCAPI_REGISTER_HMAC_ALL
 152    #define LINUXKM_LKCAPI_REGISTER_SHA1_HMAC
 153    #define LINUXKM_LKCAPI_REGISTER_SHA2_HMAC
 154    #define LINUXKM_LKCAPI_REGISTER_SHA3_HMAC
 155#endif
 156
 157#ifdef LINUXKM_LKCAPI_DONT_REGISTER_HMAC_ALL
 158    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA1_HMAC
 159    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA2_HMAC
 160    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA3_HMAC
 161#endif
 162
 163#ifdef LINUXKM_LKCAPI_REGISTER_SHA2
 164    #define LINUXKM_LKCAPI_REGISTER_SHA2_224
 165    #define LINUXKM_LKCAPI_REGISTER_SHA2_256
 166    #define LINUXKM_LKCAPI_REGISTER_SHA2_384
 167    #define LINUXKM_LKCAPI_REGISTER_SHA2_512
 168#endif
 169
 170#ifdef LINUXKM_LKCAPI_DONT_REGISTER_SHA2
 171    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA2_224
 172    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA2_256
 173    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA2_384
 174    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA2_512
 175#endif
 176
 177#ifdef LINUXKM_LKCAPI_REGISTER_SHA2_HMAC
 178    #define LINUXKM_LKCAPI_REGISTER_SHA2_224_HMAC
 179    #define LINUXKM_LKCAPI_REGISTER_SHA2_256_HMAC
 180    #define LINUXKM_LKCAPI_REGISTER_SHA2_384_HMAC
 181    #define LINUXKM_LKCAPI_REGISTER_SHA2_512_HMAC
 182#endif
 183
 184#ifdef LINUXKM_LKCAPI_DONT_REGISTER_SHA2_HMAC
 185    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA2_224_HMAC
 186    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA2_256_HMAC
 187    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA2_384_HMAC
 188    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA2_512_HMAC
 189#endif
 190
 191#ifdef LINUXKM_LKCAPI_REGISTER_SHA3
 192    #define LINUXKM_LKCAPI_REGISTER_SHA3_224
 193    #define LINUXKM_LKCAPI_REGISTER_SHA3_256
 194    #define LINUXKM_LKCAPI_REGISTER_SHA3_384
 195    #define LINUXKM_LKCAPI_REGISTER_SHA3_512
 196#endif
 197
 198#ifdef LINUXKM_LKCAPI_DONT_REGISTER_SHA3
 199    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA3_224
 200    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA3_256
 201    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA3_384
 202    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA3_512
 203#endif
 204
 205#ifdef LINUXKM_LKCAPI_REGISTER_SHA3_HMAC
 206    #define LINUXKM_LKCAPI_REGISTER_SHA3_224_HMAC
 207    #define LINUXKM_LKCAPI_REGISTER_SHA3_256_HMAC
 208    #define LINUXKM_LKCAPI_REGISTER_SHA3_384_HMAC
 209    #define LINUXKM_LKCAPI_REGISTER_SHA3_512_HMAC
 210#endif
 211
 212#ifdef LINUXKM_LKCAPI_DONT_REGISTER_SHA3_HMAC
 213    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA3_224_HMAC
 214    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA3_256_HMAC
 215    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA3_384_HMAC
 216    #define LINUXKM_LKCAPI_DONT_REGISTER_SHA3_512_HMAC
 217#endif
 218
 219#if defined(NO_HMAC) && defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_HMAC) && \
 220    !defined(LINUXKM_LKCAPI_DONT_REGISTER_HMAC_ALL)
 221    #error Config conflict: target kernel has CONFIG_CRYPTO_HMAC, but module has NO_HMAC
 222#endif
 223
 224#ifndef NO_SHA
 225    #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 226         (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_SHA1))) && \
 227        !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA1) && \
 228        !defined(LINUXKM_LKCAPI_REGISTER_SHA1)
 229        #define LINUXKM_LKCAPI_REGISTER_SHA1
 230    #endif
 231    #ifdef NO_HMAC
 232        #undef LINUXKM_LKCAPI_REGISTER_SHA1_HMAC
 233    #elif (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 234           (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_SHA1))) && \
 235          !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA1_HMAC) && \
 236          !defined(LINUXKM_LKCAPI_REGISTER_SHA1_HMAC)
 237        #define LINUXKM_LKCAPI_REGISTER_SHA1_HMAC
 238    #endif
 239#else
 240    #if defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_SHA1) && \
 241        !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA1)
 242        #error Config conflict: target kernel has CONFIG_CRYPTO_SHA1, but module has NO_SHA
 243    #endif
 244
 245    #undef LINUXKM_LKCAPI_REGISTER_SHA1
 246    #undef LINUXKM_LKCAPI_REGISTER_SHA1_HMAC
 247#endif
 248
 249#ifdef WOLFSSL_SHA224
 250    #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 251         (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_SHA256))) && \
 252        !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA2_224) &&            \
 253        !defined(LINUXKM_LKCAPI_REGISTER_SHA2_224)
 254        #define LINUXKM_LKCAPI_REGISTER_SHA2_224
 255    #endif
 256    #ifdef NO_HMAC
 257        #undef LINUXKM_LKCAPI_REGISTER_SHA2_224_HMAC
 258    #elif (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 259           (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_SHA256))) && \
 260          !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA2_224_HMAC) &&        \
 261          !defined(LINUXKM_LKCAPI_REGISTER_SHA2_224_HMAC)
 262        #define LINUXKM_LKCAPI_REGISTER_SHA2_224_HMAC
 263    #endif
 264#else
 265    #if defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_SHA256) && \
 266        !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA2_224)
 267        #error Config conflict: target kernel has CONFIG_CRYPTO_SHA256, but module is missing WOLFSSL_SHA224
 268    #endif
 269
 270    #undef LINUXKM_LKCAPI_REGISTER_SHA2_224
 271    #undef LINUXKM_LKCAPI_REGISTER_SHA2_224_HMAC
 272#endif
 273
 274#ifndef NO_SHA256
 275    #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 276         (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_SHA256))) && \
 277        !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA2_256) && \
 278        !defined(LINUXKM_LKCAPI_REGISTER_SHA2_256)
 279        #define LINUXKM_LKCAPI_REGISTER_SHA2_256
 280    #endif
 281    #ifdef NO_HMAC
 282        #undef LINUXKM_LKCAPI_REGISTER_SHA2_256_HMAC
 283    #elif (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 284           (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_SHA256))) && \
 285          !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA2_256_HMAC) &&  \
 286          !defined(LINUXKM_LKCAPI_REGISTER_SHA2_256_HMAC)
 287        #define LINUXKM_LKCAPI_REGISTER_SHA2_256_HMAC
 288    #endif
 289#else
 290    #if defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_SHA256) && \
 291        !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA2_256)
 292        #error Config conflict: target kernel has CONFIG_CRYPTO_SHA256, but module has NO_SHA256
 293    #endif
 294
 295    #undef LINUXKM_LKCAPI_REGISTER_SHA2_256
 296    #undef LINUXKM_LKCAPI_REGISTER_SHA2_256_HMAC
 297#endif
 298
 299#ifdef WOLFSSL_SHA384
 300    #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 301         (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_SHA512))) && \
 302        !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA2_384) && \
 303        !defined(LINUXKM_LKCAPI_REGISTER_SHA2_384)
 304        #define LINUXKM_LKCAPI_REGISTER_SHA2_384
 305    #endif
 306    #ifdef NO_HMAC
 307        #undef LINUXKM_LKCAPI_REGISTER_SHA2_384_HMAC
 308    #elif (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 309           (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_SHA512))) && \
 310          !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA2_384_HMAC) &&  \
 311          !defined(LINUXKM_LKCAPI_REGISTER_SHA2_384_HMAC)
 312        #define LINUXKM_LKCAPI_REGISTER_SHA2_384_HMAC
 313    #endif
 314#else
 315    #if defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_SHA512) && \
 316        !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA2_384)
 317        #error Config conflict: target kernel has CONFIG_CRYPTO_SHA512, but module is missing WOLFSSL_SHA384
 318    #endif
 319
 320    #undef LINUXKM_LKCAPI_REGISTER_SHA2_384
 321    #undef LINUXKM_LKCAPI_REGISTER_SHA2_384_HMAC
 322#endif
 323
 324#ifdef WOLFSSL_SHA512
 325    #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 326         (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_SHA512))) && \
 327        !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA2_512) && \
 328        !defined(LINUXKM_LKCAPI_REGISTER_SHA2_512)
 329        #define LINUXKM_LKCAPI_REGISTER_SHA2_512
 330    #endif
 331    #ifdef NO_HMAC
 332        #undef LINUXKM_LKCAPI_REGISTER_SHA2_512_HMAC
 333    #elif (defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 334           (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_SHA512))) && \
 335          !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA2_512_HMAC) &&  \
 336          !defined(LINUXKM_LKCAPI_REGISTER_SHA2_512_HMAC)
 337        #define LINUXKM_LKCAPI_REGISTER_SHA2_512_HMAC
 338    #endif
 339#else
 340    #if defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_SHA512) && \
 341        !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA2_512)
 342        #error Config conflict: target kernel has CONFIG_CRYPTO_SHA512, but module is missing WOLFSSL_SHA512
 343    #endif
 344
 345    #undef LINUXKM_LKCAPI_REGISTER_SHA2_512
 346    #undef LINUXKM_LKCAPI_REGISTER_SHA2_512_HMAC
 347#endif
 348
 349#ifdef WOLFSSL_SHA3
 350    #if defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 351        (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_SHA3))
 352        #if !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA3_224) && \
 353            !defined(LINUXKM_LKCAPI_REGISTER_SHA3_224)
 354            #define LINUXKM_LKCAPI_REGISTER_SHA3_224
 355        #endif
 356        #if !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA3_256) && \
 357            !defined(LINUXKM_LKCAPI_REGISTER_SHA3_256)
 358            #define LINUXKM_LKCAPI_REGISTER_SHA3_256
 359        #endif
 360        #if !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA3_384) && \
 361            !defined(LINUXKM_LKCAPI_REGISTER_SHA3_384)
 362            #define LINUXKM_LKCAPI_REGISTER_SHA3_384
 363        #endif
 364        #if !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA3_512) && \
 365            !defined(LINUXKM_LKCAPI_REGISTER_SHA3_512)
 366            #define LINUXKM_LKCAPI_REGISTER_SHA3_512
 367        #endif
 368    #endif
 369    #ifdef NO_HMAC
 370        #undef LINUXKM_LKCAPI_REGISTER_SHA3_224_HMAC
 371        #undef LINUXKM_LKCAPI_REGISTER_SHA3_256_HMAC
 372        #undef LINUXKM_LKCAPI_REGISTER_SHA3_384_HMAC
 373        #undef LINUXKM_LKCAPI_REGISTER_SHA3_512_HMAC
 374    #elif defined(LINUXKM_LKCAPI_REGISTER_ALL) || \
 375        (defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_SHA3))
 376        #if !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA3_224_HMAC) && \
 377            !defined(LINUXKM_LKCAPI_REGISTER_SHA3_224_HMAC)
 378            #define LINUXKM_LKCAPI_REGISTER_SHA3_224_HMAC
 379        #endif
 380        #if !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA3_256_HMAC) && \
 381            !defined(LINUXKM_LKCAPI_REGISTER_SHA3_256_HMAC)
 382            #define LINUXKM_LKCAPI_REGISTER_SHA3_256_HMAC
 383        #endif
 384        #if !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA3_384_HMAC) && \
 385            !defined(LINUXKM_LKCAPI_REGISTER_SHA3_384_HMAC)
 386            #define LINUXKM_LKCAPI_REGISTER_SHA3_384_HMAC
 387        #endif
 388        #if !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA3_512_HMAC) && \
 389            !defined(LINUXKM_LKCAPI_REGISTER_SHA3_512_HMAC)
 390            #define LINUXKM_LKCAPI_REGISTER_SHA3_512_HMAC
 391        #endif
 392    #endif
 393#else
 394    #if defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_SHA3) && \
 395        !defined(LINUXKM_LKCAPI_DONT_REGISTER_SHA3)
 396        #error Config conflict: target kernel has CONFIG_CRYPTO_SHA3, but module is missing WOLFSSL_SHA3
 397    #endif
 398
 399    #undef LINUXKM_LKCAPI_REGISTER_SHA3_224
 400    #undef LINUXKM_LKCAPI_REGISTER_SHA3_256
 401    #undef LINUXKM_LKCAPI_REGISTER_SHA3_384
 402    #undef LINUXKM_LKCAPI_REGISTER_SHA3_512
 403    #undef LINUXKM_LKCAPI_REGISTER_SHA3_224_HMAC
 404    #undef LINUXKM_LKCAPI_REGISTER_SHA3_256_HMAC
 405    #undef LINUXKM_LKCAPI_REGISTER_SHA3_384_HMAC
 406    #undef LINUXKM_LKCAPI_REGISTER_SHA3_512_HMAC
 407#endif
 408
 409#if (LINUX_VERSION_CODE < KERNEL_VERSION(5, 6, 0)) &&  \
 410    (defined(LINUXKM_LKCAPI_REGISTER_SHA1_HMAC) ||     \
 411     defined(LINUXKM_LKCAPI_REGISTER_SHA2_224_HMAC) || \
 412     defined(LINUXKM_LKCAPI_REGISTER_SHA2_256_HMAC) || \
 413     defined(LINUXKM_LKCAPI_REGISTER_SHA2_384_HMAC) || \
 414     defined(LINUXKM_LKCAPI_REGISTER_SHA2_512_HMAC) || \
 415     defined(LINUXKM_LKCAPI_REGISTER_SHA3_224_HMAC) || \
 416     defined(LINUXKM_LKCAPI_REGISTER_SHA3_256_HMAC) || \
 417     defined(LINUXKM_LKCAPI_REGISTER_SHA3_384_HMAC) || \
 418     defined(LINUXKM_LKCAPI_REGISTER_SHA3_512_HMAC))
 419    #error LINUXKM_LKCAPI_REGISTER for HMACs is supported only on Linux kernel versions >= 5.6.0.
 420#endif
 421
 422#ifdef HAVE_HASHDRBG
 423    #if (defined(LINUXKM_LKCAPI_REGISTER_ALL) && !defined(LINUXKM_LKCAPI_DONT_REGISTER_HASH_DRBG)) && \
 424        !defined(LINUXKM_LKCAPI_REGISTER_HASH_DRBG)
 425        #define LINUXKM_LKCAPI_REGISTER_HASH_DRBG
 426    #endif
 427    /* setup for LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT is in linuxkm_wc_port.h */
 428#else
 429    #if defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) && defined(CONFIG_CRYPTO_DRBG) && \
 430        !defined(LINUXKM_LKCAPI_DONT_REGISTER_HASH_DRBG)
 431        #error Config conflict: target kernel has CONFIG_CRYPTO_DRBG, but module is missing HAVE_HASHDRBG
 432    #endif
 433    #undef LINUXKM_LKCAPI_REGISTER_HASH_DRBG
 434#endif
 435
 436struct km_sha_state {
 437    union {
 438#ifdef LINUXKM_LKCAPI_REGISTER_SHA1
 439        struct wc_Sha sha1_state;
 440#endif
 441#ifdef LINUXKM_LKCAPI_REGISTER_SHA2_224
 442        struct wc_Sha256 sha2_224_state;
 443#endif
 444#ifdef LINUXKM_LKCAPI_REGISTER_SHA2_256
 445        struct wc_Sha256 sha2_256_state;
 446#endif
 447#ifdef LINUXKM_LKCAPI_REGISTER_SHA2_384
 448        struct wc_Sha512 sha2_384_state;
 449#endif
 450#ifdef LINUXKM_LKCAPI_REGISTER_SHA2_512
 451        struct wc_Sha512 sha2_512_state;
 452#endif
 453
 454#ifdef LINUXKM_LKCAPI_REGISTER_SHA3_224
 455        struct wc_Sha3 *sha3_224_state;
 456#endif
 457#ifdef LINUXKM_LKCAPI_REGISTER_SHA3_256
 458        struct wc_Sha3 *sha3_256_state;
 459#endif
 460#ifdef LINUXKM_LKCAPI_REGISTER_SHA3_384
 461        struct wc_Sha3 *sha3_384_state;
 462#endif
 463#ifdef LINUXKM_LKCAPI_REGISTER_SHA3_512
 464        struct wc_Sha3 *sha3_512_state;
 465#endif
 466#ifdef WOLFSSL_SHA3
 467        void *sha3_ptr;
 468#endif
 469    };
 470};
 471
 472#ifdef WOLFSSL_SHA3
 473WC_MAYBE_UNUSED static void km_sha3_free_tstate(struct km_sha_state *t_ctx) {
 474    free(t_ctx->sha3_ptr);
 475    t_ctx->sha3_ptr = NULL;
 476}
 477
 478WC_MAYBE_UNUSED static int sha3_test_once(void) {
 479    static int once = 0;
 480    static int ret;
 481    if (! once) {
 482        ret = sha3_test();
 483        once = 1;
 484    }
 485    return ret;
 486}
 487#endif
 488
 489#define WC_LINUXKM_SHA_IMPLEMENT(name, digest_size, block_size,            \
 490                                  this_cra_name, this_cra_driver_name,     \
 491                                  init_f, update_f, final_f,               \
 492                                  free_f, test_routine)                    \
 493                                                                           \
 494                                                                           \
 495static int km_ ## name ## _init(struct shash_desc *desc) {                 \
 496    struct km_sha_state *ctx = (struct km_sha_state *)shash_desc_ctx(desc);\
 497                                                                           \
 498    int ret = init_f(&ctx-> name ## _state);                               \
 499    if (ret == 0)                                                          \
 500        return 0;                                                          \
 501    else                                                                   \
 502        return -EINVAL;                                                    \
 503}                                                                          \
 504                                                                           \
 505static int km_ ## name ## _update(struct shash_desc *desc, const u8 *data, \
 506                                  unsigned int len)                        \
 507{                                                                          \
 508    struct km_sha_state *ctx = (struct km_sha_state *)shash_desc_ctx(desc);\
 509                                                                           \
 510    int ret = update_f(&ctx-> name ## _state, data, len);                  \
 511                                                                           \
 512    if (ret == 0)                                                          \
 513        return 0;                                                          \
 514    else {                                                                 \
 515        free_f(&ctx-> name ## _state);                                     \
 516        return -EINVAL;                                                    \
 517    }                                                                      \
 518}                                                                          \
 519                                                                           \
 520static int km_ ## name ## _final(struct shash_desc *desc, u8 *out) {       \
 521    struct km_sha_state *ctx = (struct km_sha_state *)shash_desc_ctx(desc);\
 522                                                                           \
 523    int ret = final_f(&ctx-> name ## _state, out);                         \
 524                                                                           \
 525    free_f(&ctx-> name ## _state);                                         \
 526                                                                           \
 527    if (ret == 0)                                                          \
 528        return 0;                                                          \
 529    else                                                                   \
 530        return -EINVAL;                                                    \
 531}                                                                          \
 532                                                                           \
 533static int km_ ## name ## _finup(struct shash_desc *desc, const u8 *data,  \
 534                                 unsigned int len, u8 *out)                \
 535{                                                                          \
 536    struct km_sha_state *ctx = (struct km_sha_state *)shash_desc_ctx(desc);\
 537                                                                           \
 538    int ret = update_f(&ctx-> name ## _state, data, len);                  \
 539                                                                           \
 540    if (ret != 0) {                                                        \
 541        free_f(&ctx-> name ## _state);                                     \
 542        return -EINVAL;                                                    \
 543    }                                                                      \
 544                                                                           \
 545    return km_ ## name ## _final(desc, out);                               \
 546}                                                                          \
 547                                                                           \
 548static int km_ ## name ## _digest(struct shash_desc *desc, const u8 *data, \
 549                                  unsigned int len, u8 *out)               \
 550{                                                                          \
 551    int ret = km_ ## name ## _init(desc);                                  \
 552    if (ret != 0)                                                          \
 553        return ret;                                                        \
 554    return km_ ## name ## _finup(desc, data, len, out);                    \
 555}                                                                          \
 556                                                                           \
 557                                                                           \
 558static struct shash_alg name ## _alg =                                     \
 559{                                                                          \
 560    .digestsize     =       (digest_size),                                 \
 561    .init           =       km_ ## name ## _init,                          \
 562    .update         =       km_ ## name ## _update,                        \
 563    .final          =       km_ ## name ## _final,                         \
 564    .finup          =       km_ ## name ## _finup,                         \
 565    .digest         =       km_ ## name ## _digest,                        \
 566    .descsize       =       sizeof(struct km_sha_state),                   \
 567    .base           =       {                                              \
 568        .cra_name        =      (this_cra_name),                           \
 569        .cra_driver_name =      (this_cra_driver_name),                    \
 570        .cra_priority    =      WOLFSSL_LINUXKM_LKCAPI_PRIORITY,           \
 571        .cra_blocksize   =      (block_size),                              \
 572        .cra_module      =      THIS_MODULE                                \
 573    }                                                                      \
 574};                                                                         \
 575static int name ## _alg_loaded = 0;                                        \
 576                                                                           \
 577static int linuxkm_test_ ## name(void) {                                   \
 578    wc_test_ret_t ret = test_routine();                                    \
 579    if (ret >= 0)                                                          \
 580        return check_shash_driver_masking(NULL /* tfm */, this_cra_name,   \
 581                                          this_cra_driver_name);           \
 582    else {                                                                 \
 583        wc_test_render_error_message("linuxkm_test_" #name " failed: ",    \
 584                                     ret);                                 \
 585        return WC_TEST_RET_DEC_EC(ret);                                    \
 586    }                                                                      \
 587}                                                                          \
 588                                                                           \
 589struct wc_swallow_the_semicolon
 590
 591#define WC_LINUXKM_SHA3_IMPLEMENT(name, digest_size, block_size,           \
 592                                  this_cra_name, this_cra_driver_name,     \
 593                                  init_f, update_f, final_f,               \
 594                                  free_f, test_routine)                    \
 595                                                                           \
 596                                                                           \
 597static int km_ ## name ## _init(struct shash_desc *desc) {                 \
 598    struct km_sha_state *ctx = (struct km_sha_state *)shash_desc_ctx(desc);\
 599    int ret;                                                               \
 600                                                                           \
 601    ctx-> name ## _state = malloc(sizeof *ctx-> name ## _state);           \
 602    if (! ctx-> name ## _state)                                            \
 603        return -ENOMEM;                                                    \
 604    ret = init_f(ctx-> name ## _state, NULL, INVALID_DEVID);               \
 605    if (ret == 0)                                                          \
 606        return 0;                                                          \
 607    else                                                                   \
 608        return -EINVAL;                                                    \
 609}                                                                          \
 610                                                                           \
 611static int km_ ## name ## _update(struct shash_desc *desc, const u8 *data, \
 612                                  unsigned int len)                        \
 613{                                                                          \
 614    struct km_sha_state *ctx = (struct km_sha_state *)shash_desc_ctx(desc);\
 615                                                                           \
 616    int ret = update_f(ctx-> name ## _state, data, len);                   \
 617                                                                           \
 618    if (ret == 0)                                                          \
 619        return 0;                                                          \
 620    else {                                                                 \
 621        free_f(ctx-> name ## _state);                                      \
 622        km_sha3_free_tstate(ctx);                                          \
 623        return -EINVAL;                                                    \
 624    }                                                                      \
 625}                                                                          \
 626                                                                           \
 627static int km_ ## name ## _final(struct shash_desc *desc, u8 *out) {       \
 628    struct km_sha_state *ctx = (struct km_sha_state *)shash_desc_ctx(desc);\
 629                                                                           \
 630    int ret = final_f(ctx-> name ## _state, out);                          \
 631                                                                           \
 632    free_f(ctx-> name ## _state);                                          \
 633    km_sha3_free_tstate(ctx);                                              \
 634    if (ret == 0)                                                          \
 635        return 0;                                                          \
 636    else                                                                   \
 637        return -EINVAL;                                                    \
 638}                                                                          \
 639                                                                           \
 640static int km_ ## name ## _finup(struct shash_desc *desc, const u8 *data,  \
 641                                 unsigned int len, u8 *out)                \
 642{                                                                          \
 643    struct km_sha_state *ctx = (struct km_sha_state *)shash_desc_ctx(desc);\
 644                                                                           \
 645    int ret = update_f(ctx-> name ## _state, data, len);                   \
 646                                                                           \
 647    if (ret != 0) {                                                        \
 648        free_f(ctx-> name ## _state);                                      \
 649        return -EINVAL;                                                    \
 650    }                                                                      \
 651                                                                           \
 652    return km_ ## name ## _final(desc, out);                               \
 653}                                                                          \
 654                                                                           \
 655static int km_ ## name ## _digest(struct shash_desc *desc, const u8 *data, \
 656                                  unsigned int len, u8 *out)               \
 657{                                                                          \
 658    int ret = km_ ## name ## _init(desc);                                  \
 659    if (ret != 0)                                                          \
 660        return ret;                                                        \
 661    return km_ ## name ## _finup(desc, data, len, out);                    \
 662}                                                                          \
 663                                                                           \
 664static struct shash_alg name ## _alg =                                     \
 665{                                                                          \
 666    .digestsize     =       (digest_size),                                 \
 667    .init           =       km_ ## name ## _init,                          \
 668    .update         =       km_ ## name ## _update,                        \
 669    .final          =       km_ ## name ## _final,                         \
 670    .finup          =       km_ ## name ## _finup,                         \
 671    .digest         =       km_ ## name ## _digest,                        \
 672    .descsize       =       sizeof(struct km_sha_state),                   \
 673    .base           =       {                                              \
 674        .cra_name        =      (this_cra_name),                           \
 675        .cra_driver_name =      (this_cra_driver_name),                    \
 676        .cra_priority    =      WOLFSSL_LINUXKM_LKCAPI_PRIORITY,           \
 677        .cra_blocksize   =      (block_size),                              \
 678        .cra_module      =      THIS_MODULE                                \
 679    }                                                                      \
 680};                                                                         \
 681static int name ## _alg_loaded = 0;                                        \
 682                                                                           \
 683static int linuxkm_test_ ## name(void) {                                   \
 684    wc_test_ret_t ret = test_routine();                                    \
 685    if (ret >= 0)                                                          \
 686        return check_shash_driver_masking(NULL /* tfm */, this_cra_name,   \
 687                                          this_cra_driver_name);           \
 688    else {                                                                 \
 689        wc_test_render_error_message("linuxkm_test_" #name " failed: ",    \
 690                                     ret);                                 \
 691        return WC_TEST_RET_DEC_EC(ret);                                    \
 692    }                                                                      \
 693}                                                                          \
 694                                                                           \
 695struct wc_swallow_the_semicolon
 696
 697#ifdef LINUXKM_LKCAPI_REGISTER_SHA1
 698    WC_LINUXKM_SHA_IMPLEMENT(sha1, WC_SHA_DIGEST_SIZE, WC_SHA_BLOCK_SIZE,
 699                             WOLFKM_SHA1_NAME, WOLFKM_SHA1_DRIVER,
 700                             wc_InitSha, wc_ShaUpdate, wc_ShaFinal,
 701                             wc_ShaFree, sha_test);
 702#endif
 703
 704#ifdef LINUXKM_LKCAPI_REGISTER_SHA2_224
 705    WC_LINUXKM_SHA_IMPLEMENT(sha2_224, WC_SHA224_DIGEST_SIZE, WC_SHA224_BLOCK_SIZE,
 706                             WOLFKM_SHA2_224_NAME, WOLFKM_SHA2_224_DRIVER,
 707                             wc_InitSha224, wc_Sha224Update, wc_Sha224Final,
 708                             wc_Sha224Free, sha224_test);
 709#endif
 710
 711#ifdef LINUXKM_LKCAPI_REGISTER_SHA2_256
 712    WC_LINUXKM_SHA_IMPLEMENT(sha2_256, WC_SHA256_DIGEST_SIZE, WC_SHA256_BLOCK_SIZE,
 713                             WOLFKM_SHA2_256_NAME, WOLFKM_SHA2_256_DRIVER,
 714                             wc_InitSha256, wc_Sha256Update, wc_Sha256Final,
 715                             wc_Sha256Free, sha256_test);
 716#endif
 717
 718#ifdef LINUXKM_LKCAPI_REGISTER_SHA2_384
 719    WC_LINUXKM_SHA_IMPLEMENT(sha2_384, WC_SHA384_DIGEST_SIZE, WC_SHA384_BLOCK_SIZE,
 720                             WOLFKM_SHA2_384_NAME, WOLFKM_SHA2_384_DRIVER,
 721                             wc_InitSha384, wc_Sha384Update, wc_Sha384Final,
 722                             wc_Sha384Free, sha384_test);
 723#endif
 724
 725#ifdef LINUXKM_LKCAPI_REGISTER_SHA2_512
 726    WC_LINUXKM_SHA_IMPLEMENT(sha2_512, WC_SHA512_DIGEST_SIZE, WC_SHA512_BLOCK_SIZE,
 727                             WOLFKM_SHA2_512_NAME, WOLFKM_SHA2_512_DRIVER,
 728                             wc_InitSha512, wc_Sha512Update, wc_Sha512Final,
 729                             wc_Sha512Free, sha512_test);
 730#endif
 731
 732#ifdef LINUXKM_LKCAPI_REGISTER_SHA3_224
 733    WC_LINUXKM_SHA3_IMPLEMENT(sha3_224, WC_SHA3_224_DIGEST_SIZE, WC_SHA3_224_BLOCK_SIZE,
 734                             WOLFKM_SHA3_224_NAME, WOLFKM_SHA3_224_DRIVER,
 735                             wc_InitSha3_224, wc_Sha3_224_Update, wc_Sha3_224_Final,
 736                             wc_Sha3_224_Free, sha3_test_once);
 737#endif
 738
 739#ifdef LINUXKM_LKCAPI_REGISTER_SHA3_256
 740    WC_LINUXKM_SHA3_IMPLEMENT(sha3_256, WC_SHA3_256_DIGEST_SIZE, WC_SHA3_256_BLOCK_SIZE,
 741                             WOLFKM_SHA3_256_NAME, WOLFKM_SHA3_256_DRIVER,
 742                             wc_InitSha3_256, wc_Sha3_256_Update, wc_Sha3_256_Final,
 743                             wc_Sha3_256_Free, sha3_test_once);
 744#endif
 745
 746#ifdef LINUXKM_LKCAPI_REGISTER_SHA3_384
 747    WC_LINUXKM_SHA3_IMPLEMENT(sha3_384, WC_SHA3_384_DIGEST_SIZE, WC_SHA3_384_BLOCK_SIZE,
 748                             WOLFKM_SHA3_384_NAME, WOLFKM_SHA3_384_DRIVER,
 749                             wc_InitSha3_384, wc_Sha3_384_Update, wc_Sha3_384_Final,
 750                             wc_Sha3_384_Free, sha3_test_once);
 751#endif
 752
 753#ifdef LINUXKM_LKCAPI_REGISTER_SHA3_512
 754    WC_LINUXKM_SHA3_IMPLEMENT(sha3_512, WC_SHA3_512_DIGEST_SIZE, WC_SHA3_512_BLOCK_SIZE,
 755                             WOLFKM_SHA3_512_NAME, WOLFKM_SHA3_512_DRIVER,
 756                             wc_InitSha3_512, wc_Sha3_512_Update, wc_Sha3_512_Final,
 757                             wc_Sha3_512_Free, sha3_test_once);
 758#endif
 759
 760struct km_sha_hmac_pstate {
 761    struct Hmac wc_hmac;
 762};
 763struct km_sha_hmac_state {
 764    struct Hmac *wc_hmac; /* HASH_MAX_DESCSIZE is 368, but sizeof(struct Hmac) is 832 */
 765};
 766
 767#ifndef NO_HMAC
 768
 769WC_MAYBE_UNUSED static int linuxkm_hmac_setkey_common(struct crypto_shash *tfm, int type, const byte* key, word32 length)
 770{
 771    struct km_sha_hmac_pstate *p_ctx = (struct km_sha_hmac_pstate *)crypto_shash_ctx(tfm);
 772    int ret;
 773
 774#if defined(HAVE_FIPS) && (FIPS_VERSION3_LT(6, 0, 0) || defined(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS) || (LINUX_VERSION_CODE >= KERNEL_VERSION(6, 0, 0)))
 775    ret = wc_HmacSetKey(&p_ctx->wc_hmac, type, key, length);
 776#else
 777    /* kernel 5.10.x crypto manager expects FIPS-undersized keys to succeed. */
 778    ret = wc_HmacSetKey_ex(&p_ctx->wc_hmac, type, key, length, 1 /* allowFlag */);
 779#endif
 780
 781    if (ret == 0)
 782        return 0;
 783    else
 784        return -EINVAL;
 785}
 786
 787WC_MAYBE_UNUSED static void km_hmac_free_tstate(struct km_sha_hmac_state *t_ctx) {
 788    wc_HmacFree(t_ctx->wc_hmac);
 789    free(t_ctx->wc_hmac);
 790    t_ctx->wc_hmac = NULL;
 791}
 792
 793WC_MAYBE_UNUSED static int km_hmac_init_tfm(struct crypto_shash *tfm)
 794{
 795    struct km_sha_hmac_pstate *p_ctx = (struct km_sha_hmac_pstate *)crypto_shash_ctx(tfm);
 796    int ret = wc_HmacInit(&p_ctx->wc_hmac, NULL /* heap */, INVALID_DEVID);
 797    if (ret == 0)
 798        return 0;
 799    else
 800        return -EINVAL;
 801}
 802
 803WC_MAYBE_UNUSED static void km_hmac_exit_tfm(struct crypto_shash *tfm)
 804{
 805    struct km_sha_hmac_pstate *p_ctx = (struct km_sha_hmac_pstate *)crypto_shash_ctx(tfm);
 806    wc_HmacFree(&p_ctx->wc_hmac);
 807    return;
 808}
 809
 810WC_MAYBE_UNUSED static int km_hmac_init(struct shash_desc *desc) {
 811    int ret;
 812    struct km_sha_hmac_state *t_ctx = (struct km_sha_hmac_state *)shash_desc_ctx(desc);
 813    struct km_sha_hmac_pstate *p_ctx = (struct km_sha_hmac_pstate *)crypto_shash_ctx(desc->tfm);
 814
 815    t_ctx->wc_hmac = malloc(sizeof *t_ctx->wc_hmac);
 816    if (! t_ctx->wc_hmac)
 817        return -ENOMEM;
 818
 819    ret = wc_HmacCopy(&p_ctx->wc_hmac, t_ctx->wc_hmac);
 820    if (ret != 0) {
 821        free(t_ctx->wc_hmac);
 822        t_ctx->wc_hmac = NULL;
 823        return -EINVAL;
 824    }
 825
 826    return 0;
 827}
 828
 829WC_MAYBE_UNUSED static int km_hmac_update(struct shash_desc *desc, const u8 *data,
 830                          unsigned int len)
 831{
 832    struct km_sha_hmac_state *ctx = (struct km_sha_hmac_state *)shash_desc_ctx(desc);
 833
 834    int ret = wc_HmacUpdate(ctx->wc_hmac, data, len);
 835
 836    if (ret == 0)
 837        return 0;
 838    else {
 839        km_hmac_free_tstate(ctx);
 840        return -EINVAL;
 841    }
 842}
 843
 844WC_MAYBE_UNUSED static int km_hmac_final(struct shash_desc *desc, u8 *out) {
 845    struct km_sha_hmac_state *ctx = (struct km_sha_hmac_state *)shash_desc_ctx(desc);
 846
 847    int ret = wc_HmacFinal(ctx->wc_hmac, out);
 848
 849    km_hmac_free_tstate(ctx);
 850
 851    if (ret == 0)
 852        return 0;
 853    else
 854        return -EINVAL;
 855}
 856
 857WC_MAYBE_UNUSED static int km_hmac_finup(struct shash_desc *desc, const u8 *data,
 858                      unsigned int len, u8 *out)
 859{
 860    struct km_sha_hmac_state *ctx = (struct km_sha_hmac_state *)shash_desc_ctx(desc);
 861
 862    int ret = wc_HmacUpdate(ctx->wc_hmac, data, len);
 863
 864    if (ret != 0)
 865        return -EINVAL;
 866
 867    return km_hmac_final(desc, out);
 868}
 869
 870WC_MAYBE_UNUSED static int km_hmac_digest(struct shash_desc *desc, const u8 *data,
 871                      unsigned int len, u8 *out)
 872{
 873    int ret = km_hmac_init(desc);
 874    if (ret != 0)
 875        return ret;
 876    return km_hmac_finup(desc, data, len, out);
 877}
 878
 879WC_MAYBE_UNUSED static int hmac_sha3_test_once(void) {
 880    static int once = 0;
 881    static int ret;
 882    if (! once) {
 883        ret = hmac_sha3_test();
 884        once = 1;
 885    }
 886    return ret;
 887}
 888
 889#define WC_LINUXKM_HMAC_IMPLEMENT(name, id, digest_size, block_size,      \
 890                                  this_cra_name, this_cra_driver_name,    \
 891                                  test_routine)                           \
 892                                                                          \
 893static int km_ ## name ## _setkey(struct crypto_shash *tfm, const u8 *key,\
 894                                  unsigned int keylen)                    \
 895{                                                                         \
 896    return linuxkm_hmac_setkey_common(tfm, id, key, keylen);              \
 897}                                                                         \
 898                                                                          \
 899static struct shash_alg name ## _alg =                                    \
 900{                                                                         \
 901    .digestsize     =       (digest_size),                                \
 902    .init           =       km_hmac_init,                                 \
 903    .update         =       km_hmac_update,                               \
 904    .final          =       km_hmac_final,                                \
 905    .finup          =       km_hmac_finup,                                \
 906    .digest         =       km_hmac_digest,                               \
 907    .setkey         =       km_ ## name ## _setkey,                       \
 908    .init_tfm       =       km_hmac_init_tfm,                             \
 909    .exit_tfm       =       km_hmac_exit_tfm,                             \
 910    .descsize       =       sizeof(struct km_sha_hmac_state),             \
 911    .base           =       {                                             \
 912        .cra_name        =      (this_cra_name),                          \
 913        .cra_driver_name =      (this_cra_driver_name),                   \
 914        .cra_priority    =      WOLFSSL_LINUXKM_LKCAPI_PRIORITY,          \
 915        .cra_blocksize   =      (block_size),                             \
 916        .cra_ctxsize     =      sizeof(struct km_sha_hmac_pstate),        \
 917        .cra_module      =      THIS_MODULE                               \
 918    }                                                                     \
 919};                                                                        \
 920static int name ## _alg_loaded = 0;                                       \
 921                                                                          \
 922static int linuxkm_test_ ## name(void) {                                  \
 923    wc_test_ret_t ret = test_routine();                                   \
 924    if (ret >= 0)                                                         \
 925        return check_shash_driver_masking(NULL /* tfm */, this_cra_name,  \
 926                                          this_cra_driver_name);          \
 927    else {                                                                \
 928        wc_test_render_error_message("linuxkm_test_" #name " failed: ",   \
 929                                     ret);                                \
 930        return WC_TEST_RET_DEC_EC(ret);                                   \
 931    }                                                                     \
 932}                                                                         \
 933                                                                          \
 934struct wc_swallow_the_semicolon
 935
 936#endif /* !NO_HMAC */
 937
 938#ifdef LINUXKM_LKCAPI_REGISTER_SHA1_HMAC
 939    WC_LINUXKM_HMAC_IMPLEMENT(sha1_hmac, WC_SHA, WC_SHA_DIGEST_SIZE,
 940                              WC_SHA_BLOCK_SIZE, WOLFKM_SHA1_HMAC_NAME,
 941                              WOLFKM_SHA1_HMAC_DRIVER, hmac_sha_test);
 942#endif /* LINUXKM_LKCAPI_REGISTER_SHA1_HMAC */
 943
 944#ifdef LINUXKM_LKCAPI_REGISTER_SHA2_224_HMAC
 945    WC_LINUXKM_HMAC_IMPLEMENT(sha2_224_hmac, WC_SHA224, WC_SHA224_DIGEST_SIZE,
 946                              WC_SHA224_BLOCK_SIZE, WOLFKM_SHA2_224_HMAC_NAME,
 947                              WOLFKM_SHA2_224_HMAC_DRIVER, hmac_sha224_test);
 948#endif
 949
 950#ifdef LINUXKM_LKCAPI_REGISTER_SHA2_256_HMAC
 951    WC_LINUXKM_HMAC_IMPLEMENT(sha2_256_hmac, WC_SHA256, WC_SHA256_DIGEST_SIZE,
 952                              WC_SHA256_BLOCK_SIZE, WOLFKM_SHA2_256_HMAC_NAME,
 953                              WOLFKM_SHA2_256_HMAC_DRIVER, hmac_sha256_test);
 954#endif
 955
 956#ifdef LINUXKM_LKCAPI_REGISTER_SHA2_384_HMAC
 957    WC_LINUXKM_HMAC_IMPLEMENT(sha2_384_hmac, WC_SHA384, WC_SHA384_DIGEST_SIZE,
 958                              WC_SHA384_BLOCK_SIZE, WOLFKM_SHA2_384_HMAC_NAME,
 959                              WOLFKM_SHA2_384_HMAC_DRIVER, hmac_sha384_test);
 960#endif
 961
 962#ifdef LINUXKM_LKCAPI_REGISTER_SHA2_512_HMAC
 963    WC_LINUXKM_HMAC_IMPLEMENT(sha2_512_hmac, WC_SHA512, WC_SHA512_DIGEST_SIZE,
 964                              WC_SHA512_BLOCK_SIZE, WOLFKM_SHA2_512_HMAC_NAME,
 965                              WOLFKM_SHA2_512_HMAC_DRIVER, hmac_sha512_test);
 966#endif
 967
 968#ifdef LINUXKM_LKCAPI_REGISTER_SHA3_224_HMAC
 969    WC_LINUXKM_HMAC_IMPLEMENT(sha3_224_hmac, WC_SHA3_224, WC_SHA3_224_DIGEST_SIZE,
 970                              WC_SHA3_224_BLOCK_SIZE, WOLFKM_SHA3_224_HMAC_NAME,
 971                              WOLFKM_SHA3_224_HMAC_DRIVER, hmac_sha3_test_once);
 972#endif
 973
 974#ifdef LINUXKM_LKCAPI_REGISTER_SHA3_256_HMAC
 975    WC_LINUXKM_HMAC_IMPLEMENT(sha3_256_hmac, WC_SHA3_256, WC_SHA3_256_DIGEST_SIZE,
 976                              WC_SHA3_256_BLOCK_SIZE, WOLFKM_SHA3_256_HMAC_NAME,
 977                              WOLFKM_SHA3_256_HMAC_DRIVER, hmac_sha3_test_once);
 978#endif
 979
 980#ifdef LINUXKM_LKCAPI_REGISTER_SHA3_384_HMAC
 981    WC_LINUXKM_HMAC_IMPLEMENT(sha3_384_hmac, WC_SHA3_384, WC_SHA3_384_DIGEST_SIZE,
 982                              WC_SHA3_384_BLOCK_SIZE, WOLFKM_SHA3_384_HMAC_NAME,
 983                              WOLFKM_SHA3_384_HMAC_DRIVER, hmac_sha3_test_once);
 984#endif
 985
 986#ifdef LINUXKM_LKCAPI_REGISTER_SHA3_512_HMAC
 987    WC_LINUXKM_HMAC_IMPLEMENT(sha3_512_hmac, WC_SHA3_512, WC_SHA3_512_DIGEST_SIZE,
 988                              WC_SHA3_512_BLOCK_SIZE, WOLFKM_SHA3_512_HMAC_NAME,
 989                              WOLFKM_SHA3_512_HMAC_DRIVER, hmac_sha3_test_once);
 990#endif
 991
 992#ifdef LINUXKM_LKCAPI_REGISTER_HASH_DRBG
 993
 994#ifdef HAVE_ENTROPY_MEMUSE
 995    #include <wolfssl/wolfcrypt/wolfentropy.h>
 996#endif
 997#include <wolfssl/wolfcrypt/random.h>
 998#include <wolfssl/wolfcrypt/rng_bank.h>
 999
1000#ifndef WC_RNG_BANK_DEFAULT_SUPPORT
1001    #error LINUXKM_LKCAPI_REGISTER_HASH_DRBG requires WC_RNG_BANK_DEFAULT_SUPPORT.
1002#endif
1003
1004static volatile int wc_linuxkm_rng_initing_default_bank_flag = 0;
1005
1006#ifndef WC_LINUXKM_INITRNG_TIMEOUT_SEC
1007    #define WC_LINUXKM_INITRNG_TIMEOUT_SEC 30
1008#endif
1009
1010static int linuxkm_affinity_lock(void *arg) {
1011    (void)arg;
1012    if (preempt_count() != 0)
1013        return ALREADY_E;
1014#if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
1015    migrate_disable(); /* this actually makes irq_count() nonzero, so that
1016                        * DISABLE_VECTOR_REGISTERS() is superfluous, but
1017                        * don't depend on that.
1018                        */
1019#endif
1020    local_bh_disable();
1021    return 0;
1022}
1023
1024static int linuxkm_affinity_get_id(void *arg, int *id) {
1025    (void)arg;
1026    *id = raw_smp_processor_id();
1027    return 0;
1028}
1029
1030static int linuxkm_affinity_unlock(void *arg) {
1031    (void)arg;
1032    local_bh_enable();
1033#if defined(CONFIG_SMP) && (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 7, 0))
1034    migrate_enable();
1035#endif
1036    return 0;
1037}
1038
1039static int wc_linuxkm_rng_bank_init(struct wc_rng_bank *ctx)
1040{
1041    int ret;
1042    word32 flags = WC_RNG_BANK_FLAG_CAN_WAIT;
1043
1044    if (wc_linuxkm_rng_initing_default_bank_flag)
1045        flags |= WC_RNG_BANK_FLAG_NO_VECTOR_OPS;
1046
1047    ret = wc_rng_bank_init(
1048        ctx, nr_cpu_ids + 4, flags, WC_LINUXKM_INITRNG_TIMEOUT_SEC,
1049        NULL /* heap */, INVALID_DEVID);
1050
1051    if (ret == 0) {
1052        ret = wc_rng_bank_set_affinity_handlers(
1053            ctx,
1054            linuxkm_affinity_lock,
1055            linuxkm_affinity_get_id,
1056            linuxkm_affinity_unlock,
1057            NULL);
1058        if (ret == 0) {
1059            if (wc_linuxkm_rng_initing_default_bank_flag) {
1060                ret = wc_rng_bank_default_set(ctx);
1061                if (ret != 0) {
1062                    (void)wc_rng_bank_fini(ctx);
1063                    pr_err("ERROR: wc_rng_bank_default_set() in wc_linuxkm_rng_bank_init() returned err %d\n", ret);
1064                    WC_DUMP_BACKTRACE_NONDEBUG;
1065                }
1066            }
1067        }
1068        else {
1069            (void)wc_rng_bank_fini(ctx);
1070            pr_err("ERROR: wc_rng_bank_set_affinity_handlers() in wc_linuxkm_rng_bank_init() returned err %d\n", ret);
1071            WC_DUMP_BACKTRACE_NONDEBUG;
1072        }
1073    }
1074    else {
1075        pr_err("ERROR: wc_rng_bank_init() in wc_linuxkm_rng_bank_init() returned err %d\n", ret);
1076        if (ret == WC_NO_ERR_TRACE(MEMORY_E))
1077            ret = -ENOMEM;
1078        else if (ret == WC_NO_ERR_TRACE(WC_TIMEOUT_E))
1079            ret = -ETIMEDOUT;
1080        else if (ret == WC_NO_ERR_TRACE(INTERRUPTED_E))
1081            ret = -EINTR;
1082        else
1083            ret = -EINVAL;
1084    }
1085
1086    return ret;
1087}
1088
1089static int wc_linuxkm_drbg_init_tfm(struct crypto_tfm *tfm)
1090{
1091    return wc_linuxkm_rng_bank_init((struct wc_rng_bank *)crypto_tfm_ctx(tfm));
1092}
1093
1094static void wc_linuxkm_drbg_exit_tfm(struct crypto_tfm *tfm)
1095{
1096    struct wc_rng_bank *ctx = (struct wc_rng_bank *)crypto_tfm_ctx(tfm);
1097    int ret;
1098
1099    ret = wc_rng_bank_default_clear(ctx);
1100    if (ret && (ret != WC_NO_ERR_TRACE(BAD_FUNC_ARG)))
1101        pr_err("ERROR: wc_rng_bank_default_clear() in wc_linuxkm_drbg_exit_tfm() returned unexpected code %d\n", ret);
1102
1103    ret = wc_rng_bank_fini(ctx);
1104
1105    if (ret != 0)
1106        pr_err("ERROR: wc_rng_bank_fini() in wc_linuxkm_drbg_exit_tfm() returned err %d\n", ret);
1107
1108    return;
1109}
1110
1111static int wc_linuxkm_drbg_default_instance_registered = 0;
1112
1113static struct wc_rng_bank_inst *linuxkm_get_drbg(struct wc_rng_bank *ctx) {
1114    int err;
1115    struct wc_rng_bank_inst *ret;
1116    word32 flags =
1117        WC_RNG_BANK_FLAG_CAN_FAIL_OVER_INST |
1118        WC_RNG_BANK_FLAG_CAN_WAIT |
1119        WC_RNG_BANK_FLAG_PREFER_AFFINITY_INST;
1120
1121    if (preempt_count() == 0)
1122        flags |= WC_RNG_BANK_FLAG_AFFINITY_LOCK;
1123    else
1124        flags |= WC_RNG_BANK_FLAG_NO_VECTOR_OPS;
1125
1126    err = wc_rng_bank_checkout(ctx, &ret, 0, WC_LINUXKM_INITRNG_TIMEOUT_SEC, flags);
1127
1128    if (err != 0) {
1129        pr_err("ERROR: wc_rng_bank_checkout() in linuxkm_get_drbg() returned err %d.\n", err);
1130        WC_DUMP_BACKTRACE_NONDEBUG;
1131        return NULL;
1132    }
1133
1134    return ret;
1135}
1136
1137static void linuxkm_put_drbg(struct wc_rng_bank *ctx, struct wc_rng_bank_inst **drbg) {
1138    int ret = wc_rng_bank_checkin(ctx, drbg);
1139    if (ret != 0) {
1140        pr_err("ERROR: wc_rng_bank_checkin() in linuxkm_put_drbg() returned err %d.\n", ret);
1141        WC_DUMP_BACKTRACE_NONDEBUG;
1142    }
1143}
1144
1145#if defined(LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT) && defined(HAVE_HASHDRBG)
1146
1147int wc_linux_kernel_rng_is_wolfcrypt(struct crypto_rng *rng) {
1148    if (rng &&
1149        wc_linuxkm_drbg_default_instance_registered &&
1150        (rng->base.__crt_alg->cra_init == wc_linuxkm_drbg_init_tfm))
1151    {
1152        return 1;
1153    }
1154    else {
1155        return 0;
1156    }
1157}
1158
1159#ifndef WC_DRBG_BANKREF
1160    #error LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT requires WC_DRBG_BANKREF support.
1161#endif
1162
1163WC_MAYBE_UNUSED static int linuxkm_InitRng_DefaultRef(WC_RNG* rng) {
1164    struct wc_rng_bank *ctx;
1165    int ret = wc_rng_bank_default_checkout(&ctx);
1166
1167    if (ret == 0) {
1168        ret = wc_InitRng_BankRef(ctx, rng);
1169        (void)wc_rng_bank_default_checkin(&ctx);
1170        return ret;
1171    }
1172    else {
1173        pr_warn_once("WARNING: linuxkm_InitRng_DefaultRef() called with null default_wc_rng_bank; falling through to wc_InitRng().\n");
1174        return wc_InitRng(rng);
1175    }
1176
1177    __builtin_unreachable();
1178}
1179#define LKCAPI_INITRNG(rng) linuxkm_InitRng_DefaultRef(rng)
1180
1181#endif /* LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT && HAVE_HASHDRBG */
1182
1183static int wc_linuxkm_drbg_generate(struct wc_rng_bank *ctx,
1184                                    const u8 *src, unsigned int slen,
1185                                    u8 *dst, unsigned int dlen)
1186{
1187    int ret, retried = 0;
1188    struct wc_rng_bank_inst *drbg = linuxkm_get_drbg(ctx);
1189
1190    if (! drbg) {
1191        pr_err_once("BUG: linuxkm_get_drbg() failed.\n");
1192        return -EFAULT;
1193    }
1194
1195    if (slen > 0) {
1196        ret = wc_RNG_DRBG_Reseed(WC_RNG_BANK_INST_TO_RNG(drbg), src, slen);
1197        if (ret != 0) {
1198            pr_warn_once("WARNING: wc_RNG_DRBG_Reseed returned %d\n",ret);
1199            ret = -EINVAL;
1200            goto out;
1201        }
1202    }
1203
1204    for (;;) {
1205        #define RNG_MAX_BLOCK_LEN_ROUNDED (RNG_MAX_BLOCK_LEN & ~0xfU)
1206        if (dlen > RNG_MAX_BLOCK_LEN_ROUNDED) {
1207            ret = wc_RNG_GenerateBlock(WC_RNG_BANK_INST_TO_RNG(drbg), dst, RNG_MAX_BLOCK_LEN_ROUNDED);
1208            if (ret == 0) {
1209                dlen -= RNG_MAX_BLOCK_LEN_ROUNDED;
1210                dst += RNG_MAX_BLOCK_LEN_ROUNDED;
1211            }
1212        }
1213        #undef RNG_MAX_BLOCK_LEN_ROUNDED
1214        else {
1215            ret = wc_RNG_GenerateBlock(WC_RNG_BANK_INST_TO_RNG(drbg), dst, dlen);
1216            if (ret == 0)
1217                dlen = 0;
1218        }
1219
1220        if (dlen == 0)
1221            break;
1222
1223        if (ret == 0)
1224            continue;
1225
1226        if (unlikely(ret == WC_NO_ERR_TRACE(RNG_FAILURE_E)) && (! retried)) {
1227            if (slen > 0)
1228                break;
1229
1230            retried = 1;
1231
1232            ret = wc_rng_bank_inst_reinit(ctx,
1233                                          drbg,
1234                                          WC_LINUXKM_INITRNG_TIMEOUT_SEC,
1235                                          WC_RNG_BANK_FLAG_CAN_WAIT);
1236
1237            if (ret == 0) {
1238                pr_warn("WARNING: reinitialized DRBG #%d after RNG_FAILURE_E from wc_RNG_GenerateBlock().\n", raw_smp_processor_id());
1239                continue;
1240            }
1241            else {
1242                pr_warn_once("ERROR: reinitialization of DRBG #%d after RNG_FAILURE_E failed with ret %d.\n", raw_smp_processor_id(), ret);
1243                ret = -EINVAL;
1244                break;
1245            }
1246        }
1247        else {
1248            pr_warn_once("ERROR: wc_linuxkm_drbg_generate() wc_RNG_GenerateBlock returned %d.\n",ret);
1249            ret = -EINVAL;
1250            break;
1251        }
1252    }
1253
1254out:
1255
1256    linuxkm_put_drbg(ctx, &drbg);
1257
1258    return ret;
1259}
1260
1261static int wc_linuxkm_drbg_generate_tfm(struct crypto_rng *tfm,
1262                        const u8 *src, unsigned int slen,
1263                        u8 *dst, unsigned int dlen)
1264{
1265    if (tfm->base.__crt_alg->cra_init != wc_linuxkm_drbg_init_tfm)
1266    {
1267        pr_err_once("BUG: mismatched tfm.\n");
1268        return -EFAULT;
1269    }
1270
1271    return wc_linuxkm_drbg_generate((struct wc_rng_bank *)crypto_rng_ctx(tfm),
1272                                    src, slen, dst, dlen);
1273}
1274
1275static int wc_linuxkm_drbg_seed(struct wc_rng_bank *ctx,
1276                        const u8 *seed, unsigned int slen)
1277{
1278    int ret;
1279
1280    if (slen == 0)
1281        return 0;
1282
1283    ret = wc_rng_bank_seed(ctx, seed, slen, WC_LINUXKM_INITRNG_TIMEOUT_SEC, WC_RNG_BANK_FLAG_CAN_WAIT);
1284    if (ret != 0) {
1285        pr_err("wc_rng_bank_seed() in wc_linuxkm_drbg_seed() returned err %d.\n", ret);
1286        ret = -EINVAL;
1287    }
1288
1289    return ret;
1290}
1291
1292static int wc_linuxkm_drbg_seed_tfm(struct crypto_rng *tfm,
1293                                    const u8 *seed, unsigned int slen)
1294{
1295    if (tfm->base.__crt_alg->cra_init != wc_linuxkm_drbg_init_tfm)
1296    {
1297        pr_err_once("BUG: mismatched tfm.\n");
1298        return -EFAULT;
1299    }
1300
1301    return wc_linuxkm_drbg_seed((struct wc_rng_bank *)crypto_rng_ctx(tfm),
1302                                seed, slen);
1303}
1304
1305static struct rng_alg wc_linuxkm_drbg = {
1306    .generate = wc_linuxkm_drbg_generate_tfm,
1307    .seed =     wc_linuxkm_drbg_seed_tfm,
1308    .seedsize = 0,
1309    .base           =       {
1310        .cra_name        =      WOLFKM_STDRNG_NAME,
1311        .cra_driver_name =      WOLFKM_STDRNG_DRIVER,
1312        .cra_priority    =      WOLFSSL_LINUXKM_LKCAPI_PRIORITY,
1313        .cra_ctxsize     =      sizeof(struct wc_rng_bank),
1314        .cra_init        =      wc_linuxkm_drbg_init_tfm,
1315        .cra_exit        =      wc_linuxkm_drbg_exit_tfm,
1316        .cra_module      =      THIS_MODULE
1317    }
1318};
1319static int wc_linuxkm_drbg_loaded = 0;
1320
1321#ifdef LINUXKM_DRBG_GET_RANDOM_BYTES
1322
1323#ifndef WOLFSSL_SMALL_STACK_CACHE
1324    /* WOLFSSL_SMALL_STACK_CACHE eliminates post-init heap allocations in SHA-2
1325     * and the Hash DRBG, fixing circular call dependencies between
1326     * get_random_u32() from kernel heap and wolfCrypt DRBG.
1327     */
1328    #error LINUXKM_DRBG_GET_RANDOM_BYTES requires WOLFSSL_SMALL_STACK_CACHE.
1329#endif
1330
1331#if !(defined(HAVE_ENTROPY_MEMUSE) || defined(HAVE_INTEL_RDSEED) ||    \
1332      defined(HAVE_AMD_RDSEED) || defined(WC_LINUXKM_RDSEED_IN_GLUE_LAYER))
1333    #error LINUXKM_DRBG_GET_RANDOM_BYTES requires a native or intrinsic entropy source.
1334#endif
1335
1336#if defined(WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS) && defined(WOLFSSL_LINUXKM_USE_GET_RANDOM_KPROBES)
1337    #error Conflicting callback model for LINUXKM_DRBG_GET_RANDOM_BYTES.
1338#endif
1339
1340#ifdef WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS
1341
1342static int wc__get_random_bytes(void *buf, size_t len)
1343{
1344    struct wc_rng_bank *current_default_wc_rng_bank;
1345    int ret = wc_rng_bank_default_checkout(&current_default_wc_rng_bank);
1346    if (ret) {
1347#ifdef WC_VERBOSE_RNG
1348        pr_err_ratelimited("ERROR: wc_rng_bank_default_checkout() in wc__get_random_bytes() returned %d.\n", ret);
1349#endif
1350        return -EFAULT;
1351    }
1352    else {
1353        ret = wc_linuxkm_drbg_generate(current_default_wc_rng_bank,
1354                                           NULL, 0, buf, len);
1355        (void)wc_rng_bank_default_checkin(&current_default_wc_rng_bank);
1356        if (ret) {
1357            pr_warn("BUG: wc__get_random_bytes falling through to native get_random_bytes with wc_linuxkm_drbg_default_instance_registered, ret=%d.\n", ret);
1358        }
1359        return ret;
1360    }
1361    __builtin_unreachable();
1362}
1363
1364/* used by kernel >=5.14.0 */
1365static ssize_t wc_get_random_bytes_user(struct iov_iter *iter) {
1366    struct wc_rng_bank *current_default_wc_rng_bank;
1367    int ret;
1368    if (unlikely(!iov_iter_count(iter)))
1369        return 0;
1370
1371    ret = wc_rng_bank_default_checkout(&current_default_wc_rng_bank);
1372    if (ret) {
1373#ifdef WC_VERBOSE_RNG
1374        pr_err_ratelimited("ERROR: wc_rng_bank_default_checkout() in wc_get_random_bytes_user() returned %d.\n", ret);
1375#endif
1376        return -ECANCELED;
1377    }
1378    else {
1379        size_t this_copied, total_copied = 0;
1380        byte block[WC_SHA256_BLOCK_SIZE];
1381
1382        for (;;) {
1383            ret = wc_linuxkm_drbg_generate(current_default_wc_rng_bank,
1384                                           NULL, 0, block, sizeof block);
1385            if (unlikely(ret != 0)) {
1386                pr_err("ERROR: wc_get_random_bytes_user() wc_linuxkm_drbg_generate() returned %d.\n", ret);
1387                break;
1388            }
1389
1390            /* note copy_to_iter() cannot be safely executed with
1391             * DISABLE_VECTOR_REGISTERS() or kprobes status, i.e.
1392             * irq_count() must be zero here.
1393             */
1394            this_copied = copy_to_iter(block, sizeof(block), iter);
1395            total_copied += this_copied;
1396            if (!iov_iter_count(iter) || this_copied != sizeof(block))
1397                break;
1398
1399            wc_static_assert(PAGE_SIZE % sizeof(block) == 0);
1400            if (total_copied % PAGE_SIZE == 0) {
1401                if (signal_pending(current))
1402                    break;
1403                cond_resched();
1404            }
1405        }
1406
1407        (void)wc_rng_bank_default_checkin(&current_default_wc_rng_bank);
1408
1409        ForceZero(block, sizeof(block));
1410
1411        if (total_copied == 0) {
1412            if (ret == 0)
1413                ret = -EFAULT;
1414            else
1415                ret = -ECANCELED;
1416        }
1417
1418        if (ret == 0)
1419            ret = (ssize_t)total_copied;
1420
1421        return ret;
1422    }
1423    __builtin_unreachable();
1424}
1425
1426/* used by kernel 4.9.0-5.13.x */
1427static ssize_t wc_extract_crng_user(void __user *buf, size_t nbytes) {
1428    int ret;
1429    struct wc_rng_bank *current_default_wc_rng_bank;
1430    if (unlikely(!nbytes))
1431        return 0;
1432
1433    ret = wc_rng_bank_default_checkout(&current_default_wc_rng_bank);
1434    if (ret) {
1435#ifdef WC_VERBOSE_RNG
1436        pr_err_ratelimited("ERROR: wc_rng_bank_default_checkout() in wc_extract_crng_user() returned %d.\n", ret);
1437#endif
1438        return -ECANCELED;
1439    }
1440    else {
1441        size_t this_copied, total_copied = 0;
1442        byte block[WC_SHA256_BLOCK_SIZE];
1443
1444        for (;;) {
1445            ret = wc_linuxkm_drbg_generate(current_default_wc_rng_bank,
1446                                           NULL, 0, block, sizeof block);
1447            if (unlikely(ret != 0)) {
1448                pr_err("ERROR: wc_extract_crng_user() wc_linuxkm_drbg_generate() returned %d.\n", ret);
1449                break;
1450            }
1451
1452            this_copied = min(nbytes - total_copied, sizeof(block));
1453            if (copy_to_user((byte *)buf + total_copied, block, this_copied)) {
1454                ret = -EFAULT;
1455                break;
1456            }
1457            total_copied += this_copied;
1458            if (this_copied != sizeof(block))
1459                break;
1460
1461            wc_static_assert(PAGE_SIZE % sizeof(block) == 0);
1462            if (total_copied % PAGE_SIZE == 0) {
1463                if (signal_pending(current))
1464                    break;
1465                cond_resched();
1466            }
1467        }
1468
1469        (void)wc_rng_bank_default_checkin(&current_default_wc_rng_bank);
1470
1471        ForceZero(block, sizeof(block));
1472
1473        if ((total_copied == 0) && (ret == 0)) {
1474            ret = -ECANCELED;
1475        }
1476
1477        if (ret == 0)
1478            ret = (ssize_t)total_copied;
1479
1480        return ret;
1481    }
1482    __builtin_unreachable();
1483}
1484
1485static int wc_mix_pool_bytes(const void *buf, size_t len) {
1486    int ret;
1487    struct wc_rng_bank *ctx;
1488    size_t i;
1489    int n;
1490    int can_sleep = (preempt_count() == 0);
1491
1492    if (len == 0)
1493        return 0;
1494
1495    ret = wc_rng_bank_default_checkout(&ctx);
1496    if (ret) {
1497#ifdef WC_VERBOSE_RNG
1498        pr_err_ratelimited("ERROR: wc_rng_bank_default_checkout() in wc_mix_pool_bytes() returned %d.\n", ret);
1499#endif
1500        return -EFAULT;
1501    }
1502
1503    ret = 0;
1504
1505    for (n = ctx->n_rngs - 1; n >= 0; --n) {
1506        struct wc_rng_bank_inst *drbg;
1507
1508        int V_offset;
1509
1510        if (wc_rng_bank_checkout(ctx, &drbg, n, 0, WC_RNG_BANK_FLAG_NONE) != 0)
1511            continue;
1512
1513#ifdef WOLFSSL_DRBG_SHA512
1514        if (WC_RNG_BANK_INST_TO_RNG(drbg)->drbgType == WC_DRBG_SHA512) {
1515            for (i = 0, V_offset = 0; i < len; ++i) {
1516                ((struct DRBG_SHA512_internal *)WC_RNG_BANK_INST_TO_RNG(drbg)->drbg512)->V[V_offset++] += ((byte *)buf)[i];
1517                if (V_offset == (int)sizeof ((struct DRBG_SHA512_internal *)WC_RNG_BANK_INST_TO_RNG(drbg)->drbg512)->V)
1518                    V_offset = 0;
1519            }
1520        }
1521        else
1522#endif /* WOLFSSL_DRBG_SHA512 */
1523        {
1524            for (i = 0, V_offset = 0; i < len; ++i) {
1525                ((struct DRBG_internal *)WC_RNG_BANK_INST_TO_RNG(drbg)->drbg)->V[V_offset++] += ((byte *)buf)[i];
1526                if (V_offset == (int)sizeof ((struct DRBG_internal *)WC_RNG_BANK_INST_TO_RNG(drbg)->drbg)->V)
1527                    V_offset = 0;
1528            }
1529        }
1530
1531        wc_rng_bank_checkin(ctx, &drbg);
1532        if (can_sleep) {
1533            if (signal_pending(current)) {
1534                ret = -EINTR;
1535                break;
1536            }
1537            cond_resched();
1538        }
1539    }
1540
1541    (void)wc_rng_bank_default_checkin(&ctx);
1542
1543    return ret;
1544}
1545
1546static int wc_crng_reseed(void) {
1547    struct wc_rng_bank *ctx;
1548    int can_sleep = (preempt_count() == 0);
1549    int ret = wc_rng_bank_default_checkout(&ctx);
1550
1551    if (ret) {
1552#ifdef WC_VERBOSE_RNG
1553        pr_err_ratelimited("ERROR: wc_rng_bank_default_checkout() in wc_crng_reseed() returned %d.\n", ret);
1554#endif
1555        return -EFAULT;
1556    }
1557
1558    ret = wc_rng_bank_reseed(ctx, WC_LINUXKM_INITRNG_TIMEOUT_SEC,
1559                             can_sleep
1560                             ?
1561                             WC_RNG_BANK_FLAG_CAN_WAIT
1562                             :
1563                             WC_RNG_BANK_FLAG_NONE);
1564
1565    (void)wc_rng_bank_default_checkin(&ctx);
1566
1567    if (ret != 0) {
1568        pr_err("ERROR: wc_rng_bank_reseed() returned err %d.\n", ret);
1569        return -EINVAL;
1570    }
1571    else {
1572        return 0;
1573    }
1574}
1575
1576struct wolfssl_linuxkm_random_bytes_handlers random_bytes_handlers = {
1577    ._get_random_bytes = wc__get_random_bytes,
1578
1579    /* pass handlers for both old and new user-mode rng, and let the kernel
1580     * patch decide which one to use.
1581     */
1582    .get_random_bytes_user = wc_get_random_bytes_user,
1583    .extract_crng_user = wc_extract_crng_user,
1584
1585    .mix_pool_bytes = wc_mix_pool_bytes,
1586    /* .credit_init_bits not implemented */
1587    .crng_reseed = wc_crng_reseed
1588};
1589
1590static int wc_get_random_bytes_callbacks_installed = 0;
1591
1592#elif defined(WOLFSSL_LINUXKM_USE_GET_RANDOM_KPROBES)
1593
1594#ifndef CONFIG_KPROBES
1595    #error WOLFSSL_LINUXKM_USE_GET_RANDOM_KPROBES without CONFIG_KPROBES.
1596#endif
1597
1598#ifndef CONFIG_X86
1599    #error WOLFSSL_LINUXKM_USE_GET_RANDOM_KPROBES requires CONFIG_X86.
1600#endif
1601
1602static int wc_get_random_bytes_by_kprobe(struct kprobe *p, struct pt_regs *regs)
1603{
1604    void *buf = (void *)regs->di;
1605    size_t len = (size_t)regs->si;
1606
1607    if (wc_linuxkm_drbg_default_instance_registered) {
1608        int ret = crypto_rng_get_bytes(crypto_default_rng, buf, len);
1609        if (ret == 0) {
1610            regs->ip = (unsigned long)p->addr + p->ainsn.size;
1611            return 1; /* Handled. */
1612        }
1613        pr_warn("BUG: wc_get_random_bytes_by_kprobe falling through to native get_random_bytes with wc_linuxkm_drbg_default_instance_registered, ret=%d.\n", ret);
1614    }
1615    else
1616        pr_warn("BUG: wc_get_random_bytes_by_kprobe called without wc_linuxkm_drbg_default_instance_registered.\n");
1617
1618    /* Not handled.  Fall through to native implementation, given
1619     * that the alternative is an immediate kernel panic.
1620     *
1621     * Because we're jumping straight to the native implementation, we need to
1622     * restore the argument registers first.
1623     */
1624
1625    asm volatile (
1626        "movq %0, %%rsi\n\t"
1627        "movq %1, %%rdi\n\t"
1628        "pushq %2\n\t"       /* Push original flags */
1629        "popfq\n\t"          /* Restore flags */
1630        :
1631        : "r" (regs->si),
1632          "r" (regs->di),
1633          "r" (regs->flags)
1634        : "memory"
1635    );
1636
1637    return 0;
1638}
1639
1640static struct kprobe wc_get_random_bytes_kprobe = {
1641    .symbol_name = "get_random_bytes",
1642    .pre_handler = wc_get_random_bytes_by_kprobe,
1643};
1644static int wc_get_random_bytes_kprobe_installed = 0;
1645
1646/* note, we can't kprobe _get_random_bytes() because it's inlined. */
1647
1648#ifdef WOLFSSL_LINUXKM_USE_GET_RANDOM_USER_KRETPROBE
1649
1650#warning Interception of /dev/random, /dev/urandom, and getrandom() using \
1651    wc_get_random_bytes_user_kretprobe_enter() is known to destabilize large \
1652    one-shot reads of randomness, due to conflicts with the kretprobe run \
1653    context (uninterruptible).  In particular, cryptsetup will fail on \
1654    /dev/urandom reads.  When in doubt, patch your kernel, activating \
1655    WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS.
1656
1657struct wc_get_random_bytes_user_kretprobe_ctx {
1658    unsigned long retval;
1659};
1660
1661static int wc_get_random_bytes_user_kretprobe_enter(struct kretprobe_instance *p, struct pt_regs *regs)
1662{
1663    struct iov_iter *iter = (struct iov_iter *)regs->di;
1664    struct wc_get_random_bytes_user_kretprobe_ctx *ctx = (struct wc_get_random_bytes_user_kretprobe_ctx *)p->data;
1665
1666    int ret;
1667    size_t this_copied = (size_t)(-1L), total_copied = 0;
1668    byte block[WC_SHA256_BLOCK_SIZE];
1669
1670    if (unlikely(!wc_linuxkm_drbg_default_instance_registered)) {
1671        pr_warn("BUG: wc_get_random_bytes_user_kretprobe_enter() without wc_linuxkm_drbg_default_instance_registered.\n");
1672        ret = -ENOENT;
1673        goto out;
1674    }
1675
1676    if (unlikely(!iov_iter_count(iter))) {
1677        ret = 0;
1678        goto out;
1679    }
1680
1681    for (;;) {
1682        ret = crypto_rng_get_bytes(crypto_default_rng, block, sizeof block);
1683        if (ret != 0) {
1684            pr_err("ERROR: wc_get_random_bytes_user_kretprobe_enter() crypto_rng_get_bytes() returned %d.\n", ret);
1685            break;
1686        }
1687
1688        /* note, in a kprobe/kretprobe, this can persistently return 0 (no
1689         * progress) with nonzero iov_iter_count(iter).
1690         */
1691        this_copied = copy_to_iter(block, sizeof(block), iter);
1692
1693        total_copied += this_copied;
1694        if ((!iov_iter_count(iter)) || (this_copied != sizeof block))
1695            break;
1696
1697        wc_static_assert(PAGE_SIZE % sizeof(block) == 0);
1698        /* we are in a kprobe context here, so we can't do any scheduler ops. */
1699        #if 0
1700        if (total_copied % PAGE_SIZE == 0) {
1701            if (signal_pending(current))
1702                break;
1703            cond_resched();
1704        }
1705        #endif
1706    }
1707
1708    ForceZero(block, sizeof(block));
1709
1710    if ((total_copied == 0) && (ret == 0))
1711        total_copied = (size_t)(-EFAULT);
1712
1713out:
1714
1715    if ((ret != 0) && (this_copied == (size_t)(-1L))) {
1716        /* crypto_rng_get_bytes() failed on the first call, before any update to the iov_iter. */
1717        pr_warn("WARNING: wc_get_random_bytes_user_kretprobe_enter() falling through to native get_random_bytes_user().\n");
1718        return -EFAULT;
1719    }
1720
1721    /* if any progress was made, report that progress.  crypto_rng_get_bytes()
1722     * failing after some progress is benign.
1723     */
1724
1725    regs->ax = ctx->retval = total_copied;
1726
1727    /* skip the native get_random_bytes_user() by telling kprobes to jump
1728     * straight to the return address.
1729     */
1730    regs->ip = (unsigned long)get_kretprobe_retaddr(p);
1731
1732    /* return 0 to tell kprobes that the handler succeeded, so that
1733     * wc_get_random_bytes_user_kretprobe_exit() will be called -- fixing up the
1734     * return value (regs->ax) is necessary.
1735     */
1736    return 0;
1737}
1738
1739static int wc_get_random_bytes_user_kretprobe_exit(struct kretprobe_instance *p, struct pt_regs *regs)
1740{
1741    struct wc_get_random_bytes_user_kretprobe_ctx *ctx = (struct wc_get_random_bytes_user_kretprobe_ctx *)p->data;
1742
1743    if (unlikely(!wc_linuxkm_drbg_default_instance_registered)) {
1744        pr_warn("BUG: wc_get_random_bytes_user_kretprobe_exit without wc_linuxkm_drbg_default_instance_registered.\n");
1745        return -EFAULT;
1746    }
1747
1748    regs->ax = ctx->retval;
1749
1750    return 0;
1751}
1752
1753static struct kretprobe wc_get_random_bytes_user_kretprobe = {
1754    .kp.symbol_name = "get_random_bytes_user",
1755    .entry_handler  = wc_get_random_bytes_user_kretprobe_enter,
1756    .handler        = wc_get_random_bytes_user_kretprobe_exit,
1757    .data_size      = sizeof(struct wc_get_random_bytes_user_kretprobe_ctx)
1758};
1759static int wc_get_random_bytes_user_kretprobe_installed = 0;
1760
1761#endif /* WOLFSSL_LINUXKM_USE_GET_RANDOM_USER_KRETPROBE */
1762
1763#else /* !WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS && !(CONFIG_KPROBES && CONFIG_X86) */
1764    #error LINUXKM_DRBG_GET_RANDOM_BYTES implementation missing for target architecture/configuration.
1765#endif
1766
1767#endif /* LINUXKM_DRBG_GET_RANDOM_BYTES */
1768
1769#if defined(LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT) && \
1770    (LINUX_VERSION_CODE >= KERNEL_VERSION(7, 1, 0))
1771static struct wc_rng_bank default_bank;
1772static int default_bank_inited;
1773#endif
1774
1775static int wc_linuxkm_drbg_startup(void)
1776{
1777    int ret;
1778
1779    if (wc_linuxkm_drbg_loaded) {
1780        pr_err("ERROR: wc_linuxkm_drbg_set_default called with wc_linuxkm_drbg_loaded.\n");
1781        return -EBUSY;
1782    }
1783
1784    ret = random_test();
1785    if (ret) {
1786        pr_err("ERROR: self-test for %s failed "
1787                           "with return code %d.\n",
1788                           wc_linuxkm_drbg.base.cra_driver_name, ret);
1789        return -EINVAL;
1790    }
1791
1792    ret = crypto_register_rng(&wc_linuxkm_drbg);
1793    if (ret != 0) {
1794        pr_err("ERROR: crypto_register_rng: %d\n", ret);
1795        return ret;
1796    }
1797
1798    {
1799        struct crypto_rng *tfm = crypto_alloc_rng(wc_linuxkm_drbg.base.cra_name, 0, 0);
1800        if (IS_ERR(tfm)) {
1801            pr_err("ERROR: allocating rng algorithm %s failed: %ld\n",
1802                   wc_linuxkm_drbg.base.cra_name, PTR_ERR(tfm));
1803            ret = PTR_ERR(tfm);
1804            tfm = NULL;
1805        }
1806        else
1807            ret = 0;
1808#ifndef LINUXKM_LKCAPI_PRIORITY_ALLOW_MASKING
1809        if (! ret) {
1810            const char *actual_driver_name = crypto_tfm_alg_driver_name(crypto_rng_tfm(tfm));
1811            if (strcmp(actual_driver_name, wc_linuxkm_drbg.base.cra_driver_name)) {
1812                pr_err("ERROR: unexpected implementation for %s: %s (expected %s)\n",
1813                       wc_linuxkm_drbg.base.cra_name,
1814                       actual_driver_name,
1815                       wc_linuxkm_drbg.base.cra_driver_name);
1816                ret = -ENOENT;
1817            }
1818        }
1819#endif
1820
1821        if (! ret) {
1822            u8 buf1[16], buf2[17];
1823            int i, j;
1824
1825            memset(buf1, 0, sizeof buf1);
1826            memset(buf2, 0, sizeof buf2);
1827
1828            ret = crypto_rng_generate(tfm, NULL, 0, buf1, (unsigned int)sizeof buf1);
1829            if (! ret)
1830                ret = crypto_rng_generate(tfm, buf1, (unsigned int)sizeof buf1, buf2, (unsigned int)sizeof buf2);
1831            if (! ret) {
1832                if (memcmp(buf1, buf2, sizeof buf1) == 0)
1833                    ret = -EBADMSG;
1834            }
1835
1836            if (! ret) {
1837                /*
1838                 * Given a correctly functioning PRNG (perfectly rectangular
1839                 * PDF), There's a 94% chance that 17 random bytes will all be
1840                 * nonzero, or a 6% chance that at least one of them will be
1841                 * zero.  Iterate up to 20 times to push that 6% chance to 1.5
1842                 * E-24, an effective certainty on a functioning PRNG.  With the
1843                 * contributions from iterations on shorter blocks, the overall
1844                 * expectation of failure is 2.13 E-24.
1845                 */
1846                for (i = 1; i <= (int)sizeof buf2; ++i) {
1847                    for (j = 0; j < 20; ++j) {
1848                        memset(buf2, 0, (size_t)i);
1849                        ret = crypto_rng_generate(tfm, NULL, 0, buf2, (unsigned int)i);
1850                        if (ret)
1851                            break;
1852                        ret = -EBADMSG;
1853                        if (! memchr(buf2, 0, (size_t)i)) {
1854                            ret = 0;
1855                            break;
1856                        }
1857                    }
1858                    if (ret)
1859                        break;
1860                }
1861
1862                if (ret)
1863                    pr_err("ERROR: wc_linuxkm_drbg_startup: PRNG quality test failed, block length %d, iters %d, ret %d\n",
1864                           i, j, ret);
1865            }
1866        }
1867
1868        if (tfm)
1869            crypto_free_rng(tfm);
1870
1871        if (ret) {
1872            crypto_unregister_rng(&wc_linuxkm_drbg);
1873            return ret;
1874        }
1875
1876    }
1877
1878    wc_linuxkm_drbg_loaded = 1;
1879
1880    WOLFKM_INSTALL_NOTICE(wc_linuxkm_drbg);
1881
1882#ifdef LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT
1883    /* for the default RNG, make sure we don't cache an underlying SHA256
1884     * method that uses vector insns (forbidden from irq handlers).
1885     */
1886    wc_linuxkm_rng_initing_default_bank_flag = 1;
1887
1888#if LINUX_VERSION_CODE < KERNEL_VERSION(7, 1, 0)
1889
1890    ret = crypto_del_default_rng();
1891    if (ret) {
1892        wc_linuxkm_rng_initing_default_bank_flag = 0;
1893        pr_err("ERROR: crypto_del_default_rng returned %d\n", ret);
1894        return ret;
1895    }
1896
1897    ret = crypto_get_default_rng();
1898
1899    wc_linuxkm_rng_initing_default_bank_flag = 0;
1900
1901    if (ret) {
1902        pr_err("ERROR: crypto_get_default_rng returned %d\n", ret);
1903        return ret;
1904    }
1905
1906    {
1907        int cur_refcnt = WC_LKM_REFCOUNT_TO_INT(wc_linuxkm_drbg.base.cra_refcnt);
1908        if (cur_refcnt < 2) {
1909            pr_err("ERROR: wc_linuxkm_drbg refcnt = %d after crypto_get_default_rng()\n", cur_refcnt);
1910            crypto_put_default_rng();
1911            return -EINVAL;
1912        }
1913    }
1914
1915    if (! crypto_default_rng) {
1916        pr_err("ERROR: crypto_default_rng is null\n");
1917        crypto_put_default_rng();
1918        return -EINVAL;
1919    }
1920
1921    if (crypto_default_rng->base.__crt_alg->cra_init != wc_linuxkm_drbg_init_tfm) {
1922        pr_err("ERROR: %s NOT registered as systemwide default stdrng -- found \"%s\".\n", wc_linuxkm_drbg.base.cra_driver_name, crypto_tfm_alg_driver_name(&crypto_default_rng->base));
1923        crypto_put_default_rng();
1924        return -EINVAL;
1925    }
1926
1927    crypto_put_default_rng();
1928
1929#else /* >= 7.1.0 */
1930
1931#ifdef CONFIG_CRYPTO_FIPS
1932    if (fips_enabled) {
1933        char buf[16];
1934
1935        ret = crypto_del_default_rng();
1936        if (ret) {
1937            wc_linuxkm_rng_initing_default_bank_flag = 0;
1938            pr_err("ERROR: crypto_del_default_rng returned %d\n", ret);
1939            return ret;
1940        }
1941
1942        ret = __crypto_stdrng_get_bytes(buf, (unsigned int)sizeof buf);
1943
1944        wc_linuxkm_rng_initing_default_bank_flag = 0;
1945
1946        if (ret) {
1947            pr_err("ERROR: __crypto_stdrng_get_bytes returned %d\n", ret);
1948            return ret;
1949        }
1950    }
1951    else
1952#endif /* CONFIG_CRYPTO_FIPS */
1953    {
1954        ret = wc_linuxkm_rng_bank_init(&default_bank);
1955        wc_linuxkm_rng_initing_default_bank_flag = 0;
1956        if (ret) {
1957            pr_err("ERROR: wc_linuxkm_rng_bank_init returned %d\n", ret);
1958            return ret;
1959        }
1960        default_bank_inited = 1;
1961    }
1962
1963#endif /* >= 7.1.0 */
1964
1965    {
1966        struct wc_rng_bank *current_default_wc_rng_bank;
1967        ret = wc_rng_bank_default_checkout(&current_default_wc_rng_bank);
1968        if (ret)
1969            pr_err("ERROR: wc_rng_bank_default_checkout() after default stdrng registration returned %d\n", ret);
1970        else {
1971            ret = wc_rng_bank_default_checkin(&current_default_wc_rng_bank);
1972            if (ret)
1973                pr_err("ERROR: wc_rng_bank_default_checkin() after wc_rng_bank_default_checkout() returned %d\n", ret);
1974        }
1975        if (ret != 0) {
1976#if defined(LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT) && \
1977    (LINUX_VERSION_CODE >= KERNEL_VERSION(7, 1, 0))
1978            if (default_bank_inited) {
1979                (void)wc_rng_bank_default_clear(&default_bank);
1980                (void)wc_rng_bank_fini(&default_bank);
1981                default_bank_inited = 0;
1982            }
1983#endif
1984            return -ECANCELED;
1985        }
1986    }
1987
1988    wc_linuxkm_drbg_default_instance_registered = 1;
1989    pr_info("%s registered as systemwide default stdrng.\n", wc_linuxkm_drbg.base.cra_driver_name);
1990    pr_info("libwolfssl: to unload module, first echo 1 > /sys/module/libwolfssl/deinstall_algs\n");
1991
1992#ifdef LINUXKM_DRBG_GET_RANDOM_BYTES
1993
1994    #ifdef WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS
1995
1996    ret = wolfssl_linuxkm_register_random_bytes_handlers(
1997        THIS_MODULE,
1998        &random_bytes_handlers);
1999
2000    if (ret == 0) {
2001        wc_get_random_bytes_callbacks_installed = 1;
2002        pr_info("libwolfssl: kernel global random_bytes handlers installed.\n");
2003    }
2004    else {
2005        pr_err("ERROR: wolfssl_linuxkm_register_random_bytes_handlers() failed: %d\n", ret);
2006        return ret;
2007    }
2008
2009    #elif defined(WOLFSSL_LINUXKM_USE_GET_RANDOM_KPROBES)
2010
2011    ret = register_kprobe(&wc_get_random_bytes_kprobe);
2012    if (ret == 0) {
2013        wc_get_random_bytes_kprobe_installed = 1;
2014        pr_info("libwolfssl: wc_get_random_bytes_kprobe installed\n");
2015    }
2016    else {
2017        pr_err("ERROR: wc_get_random_bytes_kprobe installation failed: %d\n", ret);
2018        return ret;
2019    }
2020
2021    #ifdef WOLFSSL_LINUXKM_USE_GET_RANDOM_USER_KRETPROBE
2022    ret = register_kretprobe(&wc_get_random_bytes_user_kretprobe);
2023    if (ret == 0) {
2024        wc_get_random_bytes_user_kretprobe_installed = 1;
2025        pr_info("libwolfssl: wc_get_random_bytes_user_kretprobe installed\n");
2026    }
2027    else {
2028        pr_err("ERROR: wc_get_random_bytes_user_kprobe installation failed: %d\n", ret);
2029        return ret;
2030    }
2031    #endif /* WOLFSSL_LINUXKM_USE_GET_RANDOM_USER_KRETPROBE */
2032
2033    #else
2034        #error LINUXKM_DRBG_GET_RANDOM_BYTES missing installation calls.
2035    #endif
2036
2037    #ifdef DEBUG_DRBG_RESEEDS
2038    {
2039        byte scratch[4];
2040        ret = wc__get_random_bytes(scratch, sizeof(scratch));
2041        if (ret != 0) {
2042            pr_err("ERROR: wc__get_random_bytes() returned %d\n", ret);
2043            return -EINVAL;
2044        }
2045        ret = wc_mix_pool_bytes(scratch, sizeof(scratch));
2046        if (ret != 0) {
2047            pr_err("ERROR: wc_mix_pool_bytes() returned %d\n", ret);
2048            return -EINVAL;
2049        }
2050        ret = wc_crng_reseed();
2051        if (ret != 0) {
2052            pr_err("ERROR: wc_crng_reseed() returned %d\n", ret);
2053            return -EINVAL;
2054        }
2055        ret = wc__get_random_bytes(scratch, sizeof(scratch));
2056        if (ret != 0) {
2057            pr_err("ERROR: wc__get_random_bytes() returned %d\n", ret);
2058            return -EINVAL;
2059        }
2060    }
2061    #endif
2062
2063#endif /* LINUXKM_DRBG_GET_RANDOM_BYTES */
2064
2065#endif /* LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT */
2066
2067    return 0;
2068}
2069
2070static int wc_linuxkm_drbg_cleanup(void) {
2071    int cur_refcnt;
2072
2073    if (! wc_linuxkm_drbg_loaded) {
2074        pr_err("ERROR: wc_linuxkm_drbg_cleanup called with ! wc_linuxkm_drbg_loaded\n");
2075        return -EINVAL;
2076    }
2077
2078#ifdef LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT
2079    if (wc_linuxkm_drbg_default_instance_registered) {
2080        /* These deinstallations are racey, but the kernel doesn't provide any other
2081         * way.  It's written to be retryable.
2082         */
2083        int ret;
2084
2085    #ifdef LINUXKM_DRBG_GET_RANDOM_BYTES
2086
2087        /* we need to unregister the get_random_bytes handlers first to remove
2088         * the chance that a caller will race with the crypto_unregister_rng()
2089         * below.
2090         */
2091
2092        #ifdef WOLFSSL_LINUXKM_HAVE_GET_RANDOM_CALLBACKS
2093
2094        if (wc_get_random_bytes_callbacks_installed) {
2095            ret = wolfssl_linuxkm_unregister_random_bytes_handlers();
2096            if (ret != 0) {
2097                pr_err("ERROR: wolfssl_linuxkm_unregister_random_bytes_handlers returned %d\n", ret);
2098                return ret;
2099            }
2100            pr_info("libwolfssl: kernel global random_bytes handlers uninstalled\n");
2101            wc_get_random_bytes_callbacks_installed = 0;
2102        }
2103
2104        #elif defined(WOLFSSL_LINUXKM_USE_GET_RANDOM_KPROBES)
2105
2106        if (wc_get_random_bytes_kprobe_installed) {
2107            unregister_kprobe(&wc_get_random_bytes_kprobe);
2108            barrier();
2109            wc_get_random_bytes_kprobe_installed = 0;
2110            pr_info("libwolfssl: wc_get_random_bytes_kprobe uninstalled\n");
2111        }
2112        #ifdef WOLFSSL_LINUXKM_USE_GET_RANDOM_USER_KRETPROBE
2113        if (wc_get_random_bytes_user_kretprobe_installed) {
2114            unregister_kretprobe(&wc_get_random_bytes_user_kretprobe);
2115            barrier();
2116            wc_get_random_bytes_user_kretprobe_installed = 0;
2117            pr_info("libwolfssl: wc_get_random_bytes_user_kretprobe uninstalled\n");
2118        }
2119        #endif /* WOLFSSL_LINUXKM_USE_GET_RANDOM_USER_KRETPROBE */
2120
2121        #else
2122            #error LINUXKM_DRBG_GET_RANDOM_BYTES missing deinstallation calls.
2123        #endif
2124
2125    #endif /* LINUXKM_DRBG_GET_RANDOM_BYTES */
2126
2127#if LINUX_VERSION_CODE < KERNEL_VERSION(7, 1, 0)
2128        ret = crypto_del_default_rng();
2129        if (ret) {
2130            pr_err("ERROR: crypto_del_default_rng failed: %d\n", ret);
2131            return ret;
2132        }
2133#else /* >= 7.1.0 */
2134
2135#ifdef CONFIG_CRYPTO_FIPS
2136        if (fips_enabled) {
2137            ret = crypto_del_default_rng();
2138            if (ret) {
2139                pr_err("ERROR: crypto_del_default_rng failed: %d\n", ret);
2140                return ret;
2141            }
2142        }
2143        else
2144#endif /* CONFIG_CRYPTO_FIPS */
2145        if (default_bank_inited) {
2146            ret = wc_rng_bank_default_clear(&default_bank);
2147            if (ret)
2148                pr_err("ERROR: wc_rng_bank_default_clear in wc_linuxkm_drbg_cleanup failed: %d\n", ret);
2149            else {
2150                ret = wc_rng_bank_fini(&default_bank);
2151                if (ret)
2152                    pr_err("ERROR: wc_rng_bank_fini in wc_linuxkm_drbg_cleanup failed: %d\n", ret);
2153            }
2154            default_bank_inited = 0;
2155        }
2156#endif /* >= 7.1.0 */
2157
2158        wc_linuxkm_drbg_default_instance_registered = 0;
2159    }
2160#endif /* LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT */
2161
2162    cur_refcnt = WC_LKM_REFCOUNT_TO_INT(wc_linuxkm_drbg.base.cra_refcnt);
2163
2164    if (cur_refcnt != 1) {
2165        pr_err("ERROR: wc_linuxkm_drbg_cleanup called with refcnt = %d\n", cur_refcnt);
2166        return -EBUSY;
2167    }
2168
2169    crypto_unregister_rng(&wc_linuxkm_drbg);
2170
2171    if (! (wc_linuxkm_drbg.base.cra_flags & CRYPTO_ALG_DEAD)) {
2172        pr_warn("WARNING: wc_linuxkm_drbg_cleanup: after crypto_unregister_rng, wc_linuxkm_drbg isn't dead.\n");
2173        return -EBUSY;
2174    }
2175
2176    wc_linuxkm_drbg_loaded = 0;
2177
2178    return 0;
2179}
2180
2181#endif /* LINUXKM_LKCAPI_REGISTER_HASH_DRBG */
2182
2183#ifndef LKCAPI_INITRNG
2184    #define LKCAPI_INITRNG(rng) wc_InitRng(rng)
2185#endif
2186
2187#endif /* !WC_SKIP_INCLUDED_C_FILES */