luna - wolfssl/linuxkm/module

   1/* module_hooks.c -- module load/unload hooks for libwolfssl.ko
   2 *
   3 * Copyright (C) 2006-2026 wolfSSL Inc.
   4 *
   5 * This file is part of wolfSSL.
   6 *
   7 * wolfSSL is free software; you can redistribute it and/or modify
   8 * it under the terms of the GNU General Public License as published by
   9 * the Free Software Foundation; either version 3 of the License, or
  10 * (at your option) any later version.
  11 *
  12 * wolfSSL is distributed in the hope that it will be useful,
  13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 * GNU General Public License for more details.
  16 *
  17 * You should have received a copy of the GNU General Public License
  18 * along with this program; if not, write to the Free Software
  19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
  20 */
  21
  22#define WOLFSSL_LINUXKM_NEED_LINUX_CURRENT
  23
  24#include <wolfssl/wolfcrypt/libwolfssl_sources.h>
  25
  26#ifndef WOLFSSL_LICENSE
  27    #define WOLFSSL_LICENSE "GPL"
  28#endif
  29
  30#ifdef WOLFCRYPT_ONLY
  31    #include <wolfssl/version.h>
  32#else
  33    #include <wolfssl/ssl.h>
  34#endif
  35#ifdef HAVE_FIPS
  36    #ifdef USE_CONTESTMUTEX
  37        #error USE_CONTESTMUTEX is incompatible with WOLFSSL_LINUXKM
  38    #endif
  39    #ifdef WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE
  40        #include <wolfssl/wolfcrypt/hmac.h>
  41    #endif
  42    #include <wolfssl/wolfcrypt/fips_test.h>
  43#endif
  44#if !defined(NO_CRYPT_TEST) || defined(LINUXKM_LKCAPI_REGISTER)
  45    #include <wolfcrypt/test/test.h>
  46#endif
  47#ifdef HAVE_ENTROPY_MEMUSE
  48    #include <wolfssl/wolfcrypt/wolfentropy.h>
  49#endif
  50#include <wolfssl/wolfcrypt/random.h>
  51#include <wolfssl/wolfcrypt/sha256.h>
  52#ifdef NO_INLINE
  53    #include <wolfssl/wolfcrypt/misc.h>
  54#else
  55    #define WOLFSSL_MISC_INCLUDED
  56    #include <wolfcrypt/src/misc.c>
  57#endif
  58
  59#ifdef WOLFSSL_DEBUG_TRACE_ERROR_CODES
  60    enum linux_errcodes {
  61        my_EINVAL = EINVAL,
  62        my_ENOMEM = ENOMEM,
  63        my_EBADMSG = EBADMSG
  64    };
  65
  66    #undef EINVAL
  67    #undef ENOMEM
  68    #undef EBADMSG
  69
  70    #define EINVAL WC_ERR_TRACE(my_EINVAL)
  71    #define ENOMEM WC_ERR_TRACE(my_ENOMEM)
  72    #define EBADMSG WC_ERR_TRACE(my_EBADMSG)
  73#endif
  74
  75static int libwolfssl_cleanup(void) {
  76    int ret;
  77#ifdef WOLFCRYPT_ONLY
  78    ret = wolfCrypt_Cleanup();
  79    if (ret != 0)
  80        pr_err("ERROR: wolfCrypt_Cleanup() failed: %s\n", wc_GetErrorString(ret));
  81    else
  82        pr_info("wolfCrypt " LIBWOLFSSL_VERSION_STRING " cleanup complete.\n");
  83#else
  84    ret = wolfSSL_Cleanup();
  85    if (ret != WOLFSSL_SUCCESS)
  86        pr_err("ERROR: wolfSSL_Cleanup() failed: %s\n", wc_GetErrorString(ret));
  87    else
  88        pr_info("wolfSSL " LIBWOLFSSL_VERSION_STRING " cleanup complete.\n");
  89#endif
  90
  91    return ret;
  92}
  93
  94#ifdef HAVE_FIPS
  95    /* failsafe definitions for FIPS <5.3 */
  96    #ifndef FIPS_IN_CORE_DIGEST_SIZE
  97        #ifndef NO_SHA256
  98            #define FIPS_IN_CORE_DIGEST_SIZE WC_SHA256_DIGEST_SIZE
  99            #define FIPS_IN_CORE_HASH_TYPE   WC_SHA256
 100        #elif defined(WOLFSSL_SHA384)
 101            #define FIPS_IN_CORE_DIGEST_SIZE WC_SHA384_DIGEST_SIZE
 102            #define FIPS_IN_CORE_HASH_TYPE   WC_SHA384
 103        #else
 104            #error Unsupported FIPS hash alg.
 105        #endif
 106    #endif
 107
 108    #ifndef FIPS_IN_CORE_KEY_SZ
 109        #define FIPS_IN_CORE_KEY_SZ FIPS_IN_CORE_DIGEST_SIZE
 110    #endif
 111    #ifndef FIPS_IN_CORE_VERIFY_SZ
 112        #define FIPS_IN_CORE_VERIFY_SZ FIPS_IN_CORE_DIGEST_SIZE
 113    #endif
 114
 115    #if FIPS_VERSION3_GE(6,0,0) || defined(WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE)
 116        extern char verifyCore[FIPS_IN_CORE_DIGEST_SIZE*2 + 1];
 117    #endif
 118#endif
 119
 120#ifdef WC_SYM_RELOC_TABLES
 121
 122#ifdef DEBUG_LINUXKM_PIE_SUPPORT
 123
 124/* cheap portable ad-hoc hash function to confirm bitwise stability of the PIE
 125 * binary image.
 126 */
 127static unsigned int hash_span(const u8 *start, const u8 *end, unsigned int sum) {
 128    WC_SANITIZE_DISABLE();
 129    while (start < end) {
 130        unsigned int rotate_by;
 131        sum ^= *start++;
 132        rotate_by = (sum ^ (sum >> 5)) & 31;
 133        sum = (sum << rotate_by) | (sum >> (32 - rotate_by));
 134    }
 135    WC_SANITIZE_ENABLE();
 136    return sum;
 137}
 138
 139struct wc_reloc_counts reloc_counts = {};
 140
 141#endif /* DEBUG_LINUXKM_PIE_SUPPORT */
 142
 143static int set_up_wolfssl_linuxkm_pie_redirect_table(void);
 144
 145#ifdef HAVE_FIPS
 146extern const unsigned int wolfCrypt_FIPS_ro_start[];
 147extern const unsigned int wolfCrypt_FIPS_ro_end[];
 148#endif
 149
 150#endif /* WC_SYM_RELOC_TABLES */
 151
 152#ifdef HAVE_FIPS
 153static void lkmFipsCb(int ok, int err, const char* hash)
 154{
 155    if ((! ok) || (err != 0))
 156        pr_err("ERROR: libwolfssl FIPS error: %s\n", wc_GetErrorString(err));
 157    if (err == WC_NO_ERR_TRACE(IN_CORE_FIPS_E)) {
 158        if (hash) {
 159            pr_err("In-core integrity hash check failure.\n"
 160                   "Update FIPS hash with \"make module-update-fips-hash FIPS_HASH=%s\".\n",
 161                   hash);
 162        }
 163        else {
 164            pr_err("In-core integrity hash check failure.\n");
 165            pr_err("ERROR: could not compute new hash.  Contact customer support.\n");
 166        }
 167    }
 168}
 169
 170#ifdef WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE
 171static int updateFipsHash(void);
 172#endif
 173#endif /* HAVE_FIPS */
 174
 175#ifdef WOLFSSL_LINUXKM_BENCHMARKS
 176extern int wolfcrypt_benchmark_main(int argc, char** argv);
 177#endif /* WOLFSSL_LINUXKM_BENCHMARKS */
 178
 179#ifndef WOLFSSL_LINUXKM_USE_MUTEXES
 180int wc_lkm_LockMutex(wolfSSL_Mutex* m)
 181{
 182    unsigned long irq_flags;
 183    /* first, try the cheap way. */
 184    if (spin_trylock_irqsave(&m->lock, irq_flags)) {
 185        m->irq_flags = irq_flags;
 186        return 0;
 187    }
 188    if (irq_count() != 0) {
 189        /* Note, this catches calls while SAVE_VECTOR_REGISTERS()ed as
 190         * required, because in_softirq() is always true while saved,
 191         * even for WC_FPU_INHIBITED_FLAG contexts.
 192         */
 193        spin_lock_irqsave(&m->lock, irq_flags);
 194        m->irq_flags = irq_flags;
 195        return 0;
 196    }
 197    else {
 198        for (;;) {
 199            int sig_ret = wc_linuxkm_check_for_intr_signals();
 200            if (sig_ret)
 201                return sig_ret;
 202            cond_resched();
 203            if (spin_trylock_irqsave(&m->lock, irq_flags)) {
 204                m->irq_flags = irq_flags;
 205                return 0;
 206            }
 207        }
 208    }
 209    __builtin_unreachable();
 210}
 211#endif
 212
 213WC_MAYBE_UNUSED static int linuxkm_lkcapi_sysfs_install_node(struct kobj_attribute *node, int *installed_flag)
 214{
 215    if ((installed_flag == NULL) || (! *installed_flag)) {
 216        int ret = sysfs_create_file(&THIS_MODULE->mkobj.kobj, &node->attr);
 217        if (ret) {
 218            pr_err("ERROR: sysfs_create_file failed for %s: %d\n", node->attr.name, ret);
 219            return ret;
 220        }
 221        if (installed_flag)
 222            *installed_flag = 1;
 223    }
 224    return 0;
 225}
 226
 227WC_MAYBE_UNUSED static int linuxkm_lkcapi_sysfs_deinstall_node(struct kobj_attribute *node, int *installed_flag)
 228{
 229    if ((installed_flag == NULL) || *installed_flag) {
 230        sysfs_remove_file(&THIS_MODULE->mkobj.kobj, &node->attr);
 231        if (installed_flag)
 232            *installed_flag = 0;
 233    }
 234    return 0;
 235}
 236
 237#ifdef WC_LINUXKM_SUPPORT_DUMP_TO_FILE
 238static ssize_t dump_to_file(const char *path, const u8 *buf, size_t buf_len)
 239{
 240    loff_t pos = 0;
 241    struct file *fp;
 242    ssize_t ret;
 243    char tmp;
 244
 245    if (buf_len == 0) {
 246        pr_info("libwolfssl: dump_to_file() called with buf_len == 0.  Not dumping.\n");
 247        return 0;
 248    }
 249
 250    WC_SANITIZE_DISABLE();
 251#if LINUX_VERSION_CODE < KERNEL_VERSION(5,8,0)
 252    ret = probe_kernel_read(&tmp, buf, 1);
 253    if (ret == 0)
 254        ret = probe_kernel_read(&tmp, buf + buf_len - 1, 1);
 255#else
 256    ret = copy_from_kernel_nofault(&tmp, buf, 1);
 257    if (ret == 0)
 258        ret = copy_from_kernel_nofault(&tmp, buf + buf_len - 1, 1);
 259#endif
 260    WC_SANITIZE_ENABLE();
 261    if (ret != 0) {
 262        pr_err("libwolfssl: cannot safely read from buffer %px: %d\n", buf, (int)ret);
 263        return ret;
 264    }
 265
 266    fp = filp_open(path, O_WRONLY | O_CREAT, 0644);
 267    if (IS_ERR(fp)) {
 268        pr_err("libwolfssl: cannot open %s: %ld\n", path, PTR_ERR(fp));
 269        return PTR_ERR(fp);
 270    }
 271
 272    WC_SANITIZE_DISABLE();
 273#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
 274    /* kernel_write() fixed by e13ec939e9 */
 275    ret = kernel_write(fp, buf, buf_len, &pos);
 276#elif LINUX_VERSION_CODE >= KERNEL_VERSION(3, 9, 0)
 277    /* kernel_write() exported by 7bb307e894d51 */
 278    ret = kernel_write(fp, (char *)buf, buf_len, pos);
 279#else
 280    ret = vfs_write(fp, buf, buf_len, &pos);
 281#endif
 282    WC_SANITIZE_ENABLE();
 283
 284    filp_close(fp, NULL);
 285
 286    if (ret < 0)
 287        pr_err("libwolfssl: write to %s failed: %zd\n", path, ret);
 288    else if ((size_t)ret != buf_len)
 289        pr_warn("libwolfssl: short write to %s: %zd of %zu bytes\n", path, ret, buf_len);
 290
 291    return ret;
 292}
 293
 294static char *text_dump_path;
 295static char *rodata_dump_path;
 296
 297    /* indent these so they don't look like flush-left function calls. */
 298    module_param(text_dump_path,   charp, 0444);
 299    module_param(rodata_dump_path, charp, 0444);
 300
 301MODULE_PARM_DESC(text_dump_path,
 302    "Path to dump live .wolfcrypt_text section to (e.g. /tmp/wc_text.bin)");
 303MODULE_PARM_DESC(rodata_dump_path,
 304    "Path to dump live .wolfcrypt_rodata section to (e.g. /tmp/wc_rodata.bin)");
 305
 306#endif /* WC_LINUXKM_SUPPORT_DUMP_TO_FILE */
 307
 308#ifdef HAVE_FIPS
 309    static ssize_t FIPS_rerun_self_test_handler(struct kobject *kobj, struct kobj_attribute *attr,
 310                                       const char *buf, size_t count);
 311    static struct kobj_attribute FIPS_rerun_self_test_attr = __ATTR(FIPS_rerun_self_test, 0220, NULL, FIPS_rerun_self_test_handler);
 312    static int installed_sysfs_FIPS_files = 0;
 313#endif
 314
 315#ifdef LINUXKM_LKCAPI_REGISTER
 316    #include "linuxkm/lkcapi_glue.c"
 317#endif
 318
 319/* for simplicity, we use a global count to suspend signal processing while any
 320 * thread is running fipsEntry(), wolfCrypt_IntegrityTest_fips(),
 321 * linuxkm_lkcapi_register(), or linuxkm_lkcapi_unregister().  This only affects
 322 * startup dynamics and the FIPS runtime.  Once the uninterruptible routine
 323 * completes, signal handling resumes, and any still-pending signal on
 324 * continuing threads will be processed in a timely fashion.
 325 */
 326
 327static wolfSSL_Atomic_Int wc_linuxkm_sig_ignore_count = WOLFSSL_ATOMIC_INITIALIZER(0);
 328
 329int wc_linuxkm_sig_ignore_begin(void) {
 330    return wolfSSL_Atomic_Int_AddFetch(&wc_linuxkm_sig_ignore_count, 1);
 331}
 332
 333int wc_linuxkm_sig_ignore_end(void) {
 334    return wolfSSL_Atomic_Int_SubFetch(&wc_linuxkm_sig_ignore_count, 1);
 335}
 336
 337int wc_linuxkm_check_for_intr_signals(void) {
 338    static const int intr_signals[] = WC_LINUXKM_INTR_SIGNALS;
 339    if (preempt_count() != 0)
 340        return 0;
 341    if (signal_pending(current)) {
 342        int i;
 343        for (i = 0;
 344             i < (int)sizeof(intr_signals) / (int)sizeof(intr_signals[0]);
 345             ++i)
 346        {
 347            if (sigismember(&current->pending.signal, intr_signals[i])) {
 348                if (WOLFSSL_ATOMIC_LOAD(wc_linuxkm_sig_ignore_count) > 0) {
 349#ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG
 350                    pr_info("INFO: wc_linuxkm_check_for_intr_signals ignoring "
 351                            "signal %d\n", intr_signals[i]);
 352#endif
 353                        return 0;
 354                }
 355#ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG
 356                pr_info("INFO: wc_linuxkm_check_for_intr_signals returning "
 357                       "INTERRUPTED_E on signal %d\n", intr_signals[i]);
 358#endif
 359                return INTERRUPTED_E;
 360            }
 361        }
 362    }
 363    return 0;
 364}
 365
 366void wc_linuxkm_relax_long_loop(void) {
 367    #if WC_LINUXKM_MAX_NS_WITHOUT_YIELD >= 0
 368    if (preempt_count() == 0) {
 369        #if (WC_LINUXKM_MAX_NS_WITHOUT_YIELD == 0) || !defined(CONFIG_SCHED_INFO)
 370        cond_resched();
 371        #else
 372        /* note that local_clock() wraps a local_clock_noinstr() in a
 373         * preempt_disable_notrace(), which sounds expensive but isn't --
 374         * preempt_disable_notrace() is actually just a nonlocking integer
 375         * increment of current_thread_info()->preempt.count, protected only by
 376         * various compiler optimizer barriers.
 377         */
 378        u64 now = local_clock();
 379        u64 current_last_arrival = current->sched_info.last_arrival;
 380        s64 delta = (s64)(now - current_last_arrival);
 381        if (delta > WC_LINUXKM_MAX_NS_WITHOUT_YIELD) {
 382            cond_resched();
 383            /* if nothing else is runnable, cond_resched() is a no-op and
 384             * doesn't even update .last_arrival.  we could force update by
 385             * sleeping, but there's no need.  we've been nice enough by just
 386             * cond_resched()ing, and it's actually preferable to call
 387             * cond_resched() frequently once computation has looped
 388             * continuously for longer than WC_LINUXKM_MAX_NS_WITHOUT_YIELD.
 389             */
 390        }
 391        #endif
 392        return;
 393    }
 394    #endif
 395    cpu_relax();
 396}
 397
 398#if defined(WC_LINUXKM_WOLFENTROPY_IN_GLUE_LAYER)
 399
 400int wc_linuxkm_GenerateSeed_wolfEntropy(OS_Seed* os, byte* output, word32 sz)
 401{
 402    (void)os;
 403    return wc_Entropy_Get(MAX_ENTROPY_BITS, output, sz);
 404}
 405
 406#elif defined(WC_LINUXKM_RDSEED_IN_GLUE_LAYER)
 407
 408/* backported wc_GenerateSeed_IntelRD() for FIPS v5, before breakout of wolfentropy.c. */
 409
 410#include <wolfssl/wolfcrypt/cpuid.h>
 411#include <wolfssl/wolfcrypt/random.h>
 412
 413static cpuid_flags_t intel_flags = WC_CPUID_INITIALIZER;
 414static inline void wc_InitRng_IntelRD(void)
 415{
 416    cpuid_get_flags_ex(&intel_flags);
 417}
 418
 419#define INTELRD_RETRY 32
 420
 421static WC_INLINE int IntelRDseed64(word64* seed)
 422{
 423    unsigned char ok;
 424
 425    __asm__ volatile("rdseed %0; setc %1":"=r"(*seed), "=qm"(ok));
 426    return (ok) ? 0 : -1;
 427}
 428
 429/* return 0 on success */
 430static WC_INLINE int IntelRDseed64_r(word64* rnd)
 431{
 432    int iters, retry_counter;
 433    word64 buf;
 434#if defined(HAVE_AMD_RDSEED)
 435    /* See "AMD RNG ESV Public Use Document".  Version 0.7 of October 24,
 436     * 2024 specifies 0.656 to 1.312 bits of entropy per 128 bit block of
 437     * RDSEED output, depending on CPU family.
 438     *
 439     * FIPS v5 random.c sets ENTROPY_SCALE_FACTOR to 1 for
 440     * HAVE_INTEL_RDSEED.
 441     */
 442    iters = 128;
 443#elif defined(HAVE_INTEL_RDSEED)
 444    /* The value of 2 applies to Intel's RDSEED which provides about
 445     * 0.5 bits minimum of entropy per bit. The value of 4 gives a
 446     * conservative margin for FIPS.
 447     *
 448     * FIPS v5 random.c sets ENTROPY_SCALE_FACTOR to 2 for
 449     * HAVE_INTEL_RDSEED.
 450     */
 451    iters = 2;
 452#else
 453    #error WC_LINUXKM_RDSEED_IN_GLUE_LAYER requires HAVE_INTEL_RDSEED or HAVE_AMD_RDSEED
 454#endif
 455
 456    while (--iters >= 0) {
 457        for (retry_counter = 0; retry_counter < INTELRD_RETRY; retry_counter++) {
 458            if (IntelRDseed64(&buf) == 0)
 459                break;
 460        }
 461        if (retry_counter == INTELRD_RETRY)
 462            return -1;
 463        WC_SANITIZE_DISABLE();
 464        *rnd ^= buf; /* deliberately retain any garbage passed in the dest buffer. */
 465        WC_SANITIZE_ENABLE();
 466        buf = 0;
 467    }
 468    return 0;
 469}
 470
 471/* return 0 on success */
 472int wc_linuxkm_GenerateSeed_IntelRD(struct OS_Seed* os, byte* output, word32 sz)
 473{
 474    int ret;
 475    word64 rndTmp;
 476
 477    (void)os;
 478
 479    wc_InitRng_IntelRD();
 480
 481    if (!IS_INTEL_RDSEED(intel_flags)) {
 482        static wolfSSL_Atomic_Int warned_on_missing_RDSEED = WOLFSSL_ATOMIC_INITIALIZER(0);
 483        int expected_warned_on_missing_RDSEED = 0;
 484        if (wolfSSL_Atomic_Int_CompareExchange(
 485                &warned_on_missing_RDSEED, &expected_warned_on_missing_RDSEED, 1))
 486        {
 487            pr_err("ERROR: wc_linuxkm_GenerateSeed_IntelRD() called on CPU without RDSEED support.\n");
 488        }
 489        return -1;
 490    }
 491
 492    for (; (sz / sizeof(word64)) > 0; sz -= sizeof(word64),
 493                                                    output += sizeof(word64)) {
 494        ret = IntelRDseed64_r((word64*)output);
 495        if (ret != 0) {
 496#ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG
 497            pr_err("ERROR: IntelRDseed64_r() returned code %d.\n", ret);
 498#endif
 499            return ret;
 500        }
 501    }
 502    if (sz == 0)
 503        return 0;
 504
 505    /* handle unaligned remainder */
 506    ret = IntelRDseed64_r(&rndTmp);
 507    if (ret != 0) {
 508#ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG
 509        pr_err("ERROR: IntelRDseed64_r() returned code %d.\n", ret);
 510#endif
 511        return ret;
 512    }
 513
 514    XMEMCPY(output, &rndTmp, sz);
 515    wc_ForceZero(&rndTmp, sizeof(rndTmp));
 516
 517    return 0;
 518}
 519
 520#endif /* WC_LINUXKM_RDSEED_IN_GLUE_LAYER */
 521
 522#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86)
 523    #include "linuxkm/x86_vector_register_glue.c"
 524#endif
 525
 526#ifdef CONFIG_HAVE_KPROBES
 527    static WC_MAYBE_UNUSED void *my_kallsyms_lookup_name(const char *name);
 528#endif
 529
 530#ifdef FIPS_OPTEST
 531    #ifndef HAVE_FIPS
 532        #error FIPS_OPTEST requires HAVE_FIPS.
 533    #endif
 534    #ifdef LINUXKM_LKCAPI_REGISTER
 535        #error FIPS_OPTEST is not allowed with LINUXKM_LKCAPI_REGISTER.
 536    #endif
 537    extern int linuxkm_op_test_1(int argc, const char* argv[]);
 538    extern int linuxkm_op_test_wrapper(void);
 539    static wolfSSL_Atomic_Int *conTestFailure_ptr = NULL;
 540    #ifdef HAVE_WC_FIPS_OPTEST_CONTESTFAILURE_EXPORT
 541        WOLFSSL_API extern wolfSSL_Atomic_Int wc_fips_optest_conTestFailure;
 542    #endif
 543    static ssize_t FIPS_optest_trig_handler(struct kobject *kobj, struct kobj_attribute *attr,
 544                                       const char *buf, size_t count);
 545    static struct kobj_attribute FIPS_optest_trig_attr = __ATTR(FIPS_optest_run_code, 0220, NULL, FIPS_optest_trig_handler);
 546    static int installed_sysfs_FIPS_optest_trig_files = 0;
 547#endif
 548
 549#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0)
 550static int __init wolfssl_init(void)
 551#else
 552static int wolfssl_init(void)
 553#endif
 554{
 555    int ret;
 556
 557    #ifdef WC_LINUXKM_HAVE_STACK_DEBUG
 558    {
 559        unsigned long stack_usage;
 560        stack_usage = wc_linuxkm_stack_current();
 561        pr_info("STACK INFO: usage at entry to wolfssl_init(): %lu of %lu total\n", stack_usage, THREAD_SIZE);
 562        wc_linuxkm_stack_hwm_prepare(0xee);
 563    }
 564    #endif
 565
 566#ifdef WC_SYM_RELOC_TABLES
 567    ret = set_up_wolfssl_linuxkm_pie_redirect_table();
 568    if (ret < 0)
 569        return ret;
 570#endif
 571
 572#ifdef WC_LINUXKM_SUPPORT_DUMP_TO_FILE
 573
 574#ifdef WC_SYM_RELOC_TABLES
 575    if (text_dump_path) {
 576        if (dump_to_file(text_dump_path,
 577                         (u8 *)__wc_text_start,
 578                         (size_t)((uintptr_t)__wc_text_end - (uintptr_t)__wc_text_start))
 579            > 0)
 580        {
 581            pr_info("libwolfssl: dumped .wolfcrypt_text (%zu bytes) to %s.\n",
 582                    (size_t)((uintptr_t)__wc_text_end - (uintptr_t)__wc_text_start),
 583                    text_dump_path);
 584        }
 585    }
 586    if (rodata_dump_path) {
 587        if (dump_to_file(rodata_dump_path,
 588                         (u8 *)__wc_rodata_start,
 589                         (size_t)((uintptr_t)__wc_rodata_end - (uintptr_t)__wc_rodata_start))
 590            > 0)
 591        {
 592            pr_info("libwolfssl: dumped .wolfcrypt_rodata (%zu bytes) to %s.\n",
 593                    (size_t)((uintptr_t)__wc_rodata_end - (uintptr_t)__wc_rodata_start),
 594                    rodata_dump_path);
 595        }
 596    }
 597#else
 598    if ((text_dump_path != NULL) ||
 599        (rodata_dump_path != NULL))
 600    {
 601        pr_info("libwolfssl: ignoring module dump path argument(s) -- module lacks WC_SYM_RELOC_TABLES.\n");
 602    }
 603#endif
 604
 605#endif /* WC_LINUXKM_SUPPORT_DUMP_TO_FILE */
 606
 607#ifdef WC_LINUXKM_TEST_INET_PTON
 608    {
 609        const char *src;
 610        byte dst[16] = { };
 611        int pton_ret;
 612        src = "1.2.3.4";
 613        pton_ret = wc_linuxkm_inet_pton(AF_INET, src, dst);
 614        printf("pton_ret=%d src=%s dst=%d.%d.%d.%d\n", pton_ret, src, (int)dst[0], (int)dst[1], (int)dst[2], (int)dst[3]);
 615        src = "89ab::cdef";
 616        pton_ret = wc_linuxkm_inet_pton(AF_INET6, src, dst);
 617        printf("pton_ret=%d src=%s dst=%02x%02x::%02x%02x\n", pton_ret, src, (int)dst[0], (int)dst[1], (int)dst[14], (int)dst[15]);
 618    }
 619#endif /* WC_LINUXKM_TEST_INET_PTON */
 620
 621#ifdef HAVE_FIPS
 622    /* The compiled-in verifycore must be the right length, else the module
 623     * geometry will change when the correct value is passed in, destabilizing
 624     * wc_linuxkm_pie_text_reloc_tab.  It also must be the right length for the
 625     * module-update-fips-hash recipe (in-place overwrite) to work, and for
 626     * updateFipsHash() (WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE) to be safe from
 627     * overruns.
 628     */
 629    {
 630        size_t verifyCore_len;
 631#if FIPS_VERSION3_GE(6,0,0) || defined(WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE)
 632        verifyCore_len = strlen(verifyCore);
 633#else
 634#ifdef CONFIG_HAVE_KPROBES
 635        char *verifyCore_ptr = my_kallsyms_lookup_name("verifyCore");
 636        if (verifyCore_ptr)
 637            verifyCore_len = strlen(verifyCore_ptr);
 638        else
 639#endif /* CONFIG_HAVE_KPROBES */
 640        {
 641            /* can't check -- have to assume. */
 642#if defined(CONFIG_HAVE_KPROBES) && (defined(DEBUG_LINUXKM_PIE_SUPPORT) || defined(WOLFSSL_LINUXKM_VERBOSE_DEBUG))
 643            pr_err("INFO: couldn't get verifyCore_ptr -- skipping verifyCore length check.\n");
 644#endif
 645            verifyCore_len = FIPS_IN_CORE_DIGEST_SIZE*2;
 646        }
 647#endif
 648        if (verifyCore_len != FIPS_IN_CORE_DIGEST_SIZE*2) {
 649            pr_err("ERROR: compile-time FIPS hash is the wrong length (expected %d hex digits, got %zu).\n", FIPS_IN_CORE_DIGEST_SIZE*2, verifyCore_len);
 650            return -ECANCELED;
 651        }
 652    }
 653
 654#ifdef WC_SYM_RELOC_TABLES
 655    if (((uintptr_t)__wc_text_start > (uintptr_t)wolfCrypt_FIPS_first) ||
 656        ((uintptr_t)__wc_text_end < (uintptr_t)wolfCrypt_FIPS_last) ||
 657        ((uintptr_t)__wc_rodata_start > (uintptr_t)wolfCrypt_FIPS_ro_start) ||
 658        ((uintptr_t)__wc_rodata_end < (uintptr_t)wolfCrypt_FIPS_ro_end))
 659    {
 660        pr_err("ERROR: ELF segment fenceposts and FIPS fenceposts conflict.\n");
 661        return -ECANCELED;
 662    }
 663#endif
 664
 665#ifdef WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE
 666#ifdef CONFIG_MODULE_SIG
 667    if (THIS_MODULE->sig_ok == false) {
 668        pr_err("ERROR: wolfSSL module load aborted -- bad or missing module signature with FIPS dynamic hash.\n");
 669        return -ECANCELED;
 670    }
 671#endif
 672    #if defined(WC_SYM_RELOC_TABLES) && defined(DEBUG_LINUXKM_PIE_SUPPORT)
 673    reloc_counts.text = reloc_counts.rodata = reloc_counts.rwdata = reloc_counts.bss =
 674        reloc_counts.other = 0;
 675    #endif
 676    ret = updateFipsHash();
 677    if (ret < 0) {
 678        pr_err("ERROR: wolfSSL module load aborted -- updateFipsHash: %s\n",wc_GetErrorString(ret));
 679        return -ECANCELED;
 680    }
 681
 682    #if defined(WC_SYM_RELOC_TABLES) && defined(DEBUG_LINUXKM_PIE_SUPPORT)
 683    pr_info("FIPS-bounded relocation normalizations from updateFipsHash(): text=%d, rodata=%d, rwdata=%d, bss=%d, other=%d\n",
 684            reloc_counts.text, reloc_counts.rodata, reloc_counts.rwdata, reloc_counts.bss, reloc_counts.other);
 685    #endif
 686
 687#endif /* WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE */
 688
 689#endif /* HAVE_FIPS */
 690
 691#if defined(WC_SYM_RELOC_TABLES) && defined(DEBUG_LINUXKM_PIE_SUPPORT)
 692
 693#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 4, 0)
 694    /* see linux commit ac3b432839 */
 695    #define THIS_MODULE_TEXT_BASE (THIS_MODULE->mem[MOD_TEXT].base)
 696    #define THIS_MODULE_TEXT_SIZE (THIS_MODULE->mem[MOD_TEXT].size)
 697    #define THIS_MODULE_RO_BASE (THIS_MODULE->mem[MOD_RODATA].base)
 698    #define THIS_MODULE_RO_SIZE (THIS_MODULE->mem[MOD_RODATA].size)
 699#elif LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0)
 700    #define THIS_MODULE_TEXT_BASE (THIS_MODULE->core_layout.base)
 701    #define THIS_MODULE_TEXT_SIZE (THIS_MODULE->core_layout.text_size)
 702    #define THIS_MODULE_RO_BASE ((char *)THIS_MODULE->core_layout.base + THIS_MODULE->core_layout.text_size)
 703    #define THIS_MODULE_RO_SIZE (THIS_MODULE->core_layout.ro_size)
 704#else
 705    #define THIS_MODULE_TEXT_BASE (THIS_MODULE->module_core)
 706    #define THIS_MODULE_TEXT_SIZE (THIS_MODULE->core_text_size)
 707    #define THIS_MODULE_RO_BASE ((char *)THIS_MODULE->module_core + THIS_MODULE->core_ro_size)
 708    #define THIS_MODULE_RO_SIZE (THIS_MODULE->core_ro_size)
 709#endif
 710
 711    {
 712        unsigned int text_hash = hash_span((const u8 *)__wc_text_start, (const u8 *)__wc_text_end, 1);
 713        unsigned int rodata_hash = hash_span((const u8 *)__wc_rodata_start, (const u8 *)__wc_rodata_end, 1);
 714        u8 *canon_buf = malloc(WOLFSSL_SEGMENT_CANONICALIZER_BUFSIZ);
 715        ssize_t cur_reloc_index;
 716        const u8 *text_p = (const u8 *)__wc_text_start;
 717        const u8 *rodata_p = (const u8 *)__wc_rodata_start;
 718        unsigned int stabilized_text_hash = 1;
 719        unsigned int stabilized_rodata_hash = 1;
 720
 721        if (! canon_buf) {
 722            pr_err("ERROR: malloc(%d) for WOLFSSL_*_SEGMENT_CANONICALIZER failed: %ld.\n", WOLFSSL_SEGMENT_CANONICALIZER_BUFSIZ, PTR_ERR(canon_buf));
 723            return -ECANCELED;
 724        }
 725
 726        reloc_counts.text = reloc_counts.rodata = reloc_counts.rwdata = reloc_counts.bss =
 727            reloc_counts.other = 0;
 728
 729        cur_reloc_index = -1;
 730        while (text_p < (const u8 *)__wc_text_end) {
 731            size_t text_in_out_len = min(WOLFSSL_SEGMENT_CANONICALIZER_BUFSIZ,
 732                                         (size_t)((const u8 *)__wc_text_end - text_p));
 733            ssize_t progress =
 734                WOLFSSL_SEGMENT_CANONICALIZER(
 735                    text_p,
 736                    &text_in_out_len,
 737                    canon_buf, &cur_reloc_index);
 738            if (progress <= 0) {
 739                pr_err("ERROR: progress=%ld from WOLFSSL_SEGMENT_CANONICALIZER() at offset %x (text=%x-%x).\n",
 740                       (long)progress,
 741                       (unsigned)(uintptr_t)text_p,
 742                       (unsigned)(uintptr_t)__wc_text_start,
 743                       (unsigned)(uintptr_t)__wc_text_end);
 744                free(canon_buf);
 745                return -ECANCELED;
 746            }
 747            stabilized_text_hash = hash_span(canon_buf, canon_buf + text_in_out_len, stabilized_text_hash);
 748            text_p += progress;
 749        }
 750
 751        /* note verifyCore is hashed along with the rest of .rodata_wolfcrypt. */
 752        cur_reloc_index = -1;
 753        while (rodata_p < (const u8 *)__wc_rodata_end) {
 754            size_t rodata_in_out_len = min(WOLFSSL_SEGMENT_CANONICALIZER_BUFSIZ,
 755                                         (size_t)((const u8 *)__wc_rodata_end - rodata_p));
 756            ssize_t progress =
 757                WOLFSSL_SEGMENT_CANONICALIZER(
 758                    rodata_p,
 759                    &rodata_in_out_len,
 760                    canon_buf, &cur_reloc_index);
 761            if (progress <= 0) {
 762                pr_err("ERROR: progress=%ld from WOLFSSL_SEGMENT_CANONICALIZER() at offset %x (rodata=%x-%x).\n",
 763                       (long)progress,
 764                       (unsigned)(uintptr_t)rodata_p,
 765                       (unsigned)(uintptr_t)__wc_rodata_start,
 766                       (unsigned)(uintptr_t)__wc_rodata_end);
 767                free(canon_buf);
 768                return -ECANCELED;
 769            }
 770            stabilized_rodata_hash = hash_span(canon_buf, canon_buf + rodata_in_out_len, stabilized_rodata_hash);
 771            rodata_p += progress;
 772        }
 773
 774        free(canon_buf);
 775        canon_buf = 0;
 776
 777        /* note, "%pK" conceals the actual layout information.  "%px" exposes
 778         * the true module start address, which is potentially useful to an
 779         * attacker.
 780         */
 781        pr_info("wolfCrypt segment hashes (spans): text 0x%x (%llu), rodata 0x%x (%llu), offset %c0x%llx, canon text 0x%x, canon rodata 0x%x\n",
 782                text_hash, (unsigned long long)((uintptr_t)__wc_text_end - (uintptr_t)__wc_text_start),
 783                rodata_hash, (unsigned long long)((uintptr_t)__wc_rodata_end - (uintptr_t)__wc_rodata_start),
 784                (uintptr_t)__wc_text_start < (uintptr_t)&__wc_rodata_start[0] ? '+' : '-',
 785                (uintptr_t)__wc_text_start < (uintptr_t)&__wc_rodata_start[0] ? (unsigned long long)((uintptr_t)&__wc_rodata_start[0] - (uintptr_t)__wc_text_start) : (unsigned long long)((uintptr_t)__wc_text_start - (uintptr_t)&__wc_rodata_start[0]),
 786                stabilized_text_hash,
 787                stabilized_rodata_hash);
 788
 789        pr_info("wolfCrypt segments: text=%llx-%llx, rodata=%llx-%llx, "
 790                "rwdata=%llx-%llx, bss=%llx-%llx\n",
 791                (unsigned long long)(uintptr_t)__wc_text_start,
 792                (unsigned long long)(uintptr_t)__wc_text_end,
 793                (unsigned long long)(uintptr_t)__wc_rodata_start,
 794                (unsigned long long)(uintptr_t)__wc_rodata_end,
 795                (unsigned long long)(uintptr_t)__wc_rwdata_start,
 796                (unsigned long long)(uintptr_t)__wc_rwdata_end,
 797                (unsigned long long)(uintptr_t)__wc_bss_start,
 798                (unsigned long long)(uintptr_t)__wc_bss_end);
 799
 800        pr_info("whole-segment relocation normalizations: text=%d, rodata=%d, rwdata=%d, bss=%d, other=%d\n",
 801                reloc_counts.text, reloc_counts.rodata, reloc_counts.rwdata, reloc_counts.bss, reloc_counts.other);
 802    }
 803
 804#endif /* WC_SYM_RELOC_TABLES && DEBUG_LINUXKM_PIE_SUPPORT */
 805
 806#ifdef WC_LINUXKM_HAVE_STACK_DEBUG
 807    {
 808        unsigned long stack_usage;
 809        stack_usage = wc_linuxkm_stack_hwm_measure_rel(0xee);
 810        pr_info("STACK INFO: rel usage by wolfssl_init() initial setup: %lu\n", stack_usage);
 811        /* shush up false stack HWM reading by kernel: */
 812        wc_linuxkm_stack_hwm_prepare(0);
 813    }
 814#endif
 815
 816#ifdef HAVE_FIPS
 817    ret = wolfCrypt_SetCb_fips(lkmFipsCb);
 818    if (ret != 0) {
 819        pr_err("ERROR: wolfCrypt_SetCb_fips() failed: %s\n", wc_GetErrorString(ret));
 820        return -ECANCELED;
 821    }
 822
 823#if defined(WC_SYM_RELOC_TABLES) && defined(DEBUG_LINUXKM_PIE_SUPPORT)
 824    reloc_counts.text = reloc_counts.rodata = reloc_counts.rwdata = reloc_counts.bss =
 825        reloc_counts.other = 0;
 826#endif
 827
 828    if (WC_SIG_IGNORE_BEGIN() >= 0) {
 829        fipsEntry();
 830        (void)WC_SIG_IGNORE_END();
 831    }
 832    else
 833        pr_err("ERROR: WC_SIG_IGNORE_BEGIN() failed.\n");
 834
 835#if defined(WC_SYM_RELOC_TABLES) && defined(DEBUG_LINUXKM_PIE_SUPPORT)
 836    pr_info("FIPS-bounded relocation normalizations: text=%d, rodata=%d, rwdata=%d, bss=%d, other=%d\n",
 837            reloc_counts.text, reloc_counts.rodata, reloc_counts.rwdata, reloc_counts.bss, reloc_counts.other);
 838#endif
 839
 840    ret = wolfCrypt_GetStatus_fips();
 841    if (ret != 0) {
 842        pr_err("ERROR: wolfCrypt_GetStatus_fips() failed with code %d: %s\n", ret, wc_GetErrorString(ret));
 843        if (ret == WC_NO_ERR_TRACE(IN_CORE_FIPS_E)) {
 844            const char *newhash = wolfCrypt_GetCoreHash_fips();
 845            if (newhash) {
 846                pr_err("In-core integrity hash check failure.\n"
 847                       "Update FIPS hash with \"make module-update-fips-hash FIPS_HASH=%s\".\n",
 848                       newhash);
 849            }
 850            else {
 851                pr_err("In-core integrity hash check failure.\n");
 852                pr_err("ERROR: could not compute new hash.  Contact customer support.\n");
 853            }
 854        }
 855        return -ECANCELED;
 856    }
 857#endif /* HAVE_FIPS */
 858
 859#ifdef WC_RNG_SEED_CB
 860    ret = wc_SetSeed_Cb(WC_GENERATE_SEED_DEFAULT);
 861
 862    if (ret < 0) {
 863        pr_err("ERROR: wc_SetSeed_Cb() failed with return code %d.\n", ret);
 864        (void)libwolfssl_cleanup();
 865        msleep(10);
 866        return -ECANCELED;
 867    }
 868#endif /* WC_RNG_SEED_CB */
 869
 870#ifdef WOLFCRYPT_ONLY
 871    ret = wolfCrypt_Init();
 872    if (ret != 0) {
 873        pr_err("ERROR: wolfCrypt_Init() failed: %s\n", wc_GetErrorString(ret));
 874        return -ECANCELED;
 875    }
 876#else
 877    ret = wolfSSL_Init();
 878    if (ret != WOLFSSL_SUCCESS) {
 879        pr_err("ERROR: wolfSSL_Init() failed: %s\n", wc_GetErrorString(ret));
 880        return -ECANCELED;
 881    }
 882#endif
 883
 884#if defined(HAVE_FIPS) && FIPS_VERSION3_GT(5,2,0)
 885
 886    #ifdef WC_LINUXKM_HAVE_STACK_DEBUG
 887    {
 888        unsigned long stack_usage;
 889        stack_usage = wc_linuxkm_stack_current();
 890        pr_info("STACK INFO: usage at call to wc_RunAllCast_fips(): %lu of %lu total\n", stack_usage, THREAD_SIZE);
 891        wc_linuxkm_stack_hwm_prepare(0xee);
 892    #endif
 893
 894    ret = wc_RunAllCast_fips();
 895
 896#ifdef WC_LINUXKM_HAVE_STACK_DEBUG
 897        stack_usage = wc_linuxkm_stack_hwm_measure_rel(0xee);
 898        pr_info("STACK INFO: rel usage by wc_RunAllCast_fips(): %lu\n", stack_usage);
 899        /* shush up false stack HWM reading by kernel: */
 900        wc_linuxkm_stack_hwm_prepare(0);
 901    }
 902#endif
 903
 904    if (ret != 0) {
 905        pr_err("ERROR: wc_RunAllCast_fips() failed with return value %d\n", ret);
 906        return -ECANCELED;
 907    }
 908
 909    pr_info("FIPS 140-3 wolfCrypt-fips v%d.%d.%d%s%s startup "
 910            "self-test succeeded.\n",
 911#ifdef HAVE_FIPS_VERSION_MAJOR
 912            HAVE_FIPS_VERSION_MAJOR,
 913#else
 914            HAVE_FIPS_VERSION,
 915#endif
 916#ifdef HAVE_FIPS_VERSION_MINOR
 917            HAVE_FIPS_VERSION_MINOR,
 918#else
 919            0,
 920#endif
 921#ifdef HAVE_FIPS_VERSION_PATCH
 922            HAVE_FIPS_VERSION_PATCH,
 923#else
 924            0,
 925#endif
 926#ifdef HAVE_FIPS_VERSION_PORT
 927            "-",
 928            HAVE_FIPS_VERSION_PORT
 929#else
 930            "",
 931            ""
 932#endif
 933        );
 934#endif /* HAVE_FIPS && FIPS_VERSION3_GT(5,2,0) */
 935
 936#ifdef FIPS_OPTEST
 937    #ifdef HAVE_WC_FIPS_OPTEST_CONTESTFAILURE_EXPORT
 938    conTestFailure_ptr = &wc_fips_optest_conTestFailure;
 939    #else
 940    conTestFailure_ptr = (wolfSSL_Atomic_Int *)my_kallsyms_lookup_name("conTestFailure");
 941    if (conTestFailure_ptr == NULL) {
 942        pr_err("ERROR: couldn't obtain conTestFailure_ptr.\n");
 943        return -ECANCELED;
 944    }
 945    #endif
 946
 947    ret = linuxkm_lkcapi_sysfs_install_node(&FIPS_optest_trig_attr, &installed_sysfs_FIPS_optest_trig_files);
 948    if (ret != 0) {
 949        pr_err("ERROR: linuxkm_lkcapi_sysfs_install_node() failed for %s (code %d).\n", FIPS_optest_trig_attr.attr.name, ret);
 950        return -ECANCELED;
 951    }
 952
 953#ifdef FIPS_OPTEST_FULL_RUN_AT_MODULE_INIT
 954
 955    #ifdef WC_LINUXKM_HAVE_STACK_DEBUG
 956    {
 957        unsigned long stack_usage;
 958        stack_usage = wc_linuxkm_stack_current();
 959        pr_info("STACK INFO: usage at call to linuxkm_op_test_wrapper(): %lu of %lu total\n", stack_usage, THREAD_SIZE);
 960        wc_linuxkm_stack_hwm_prepare(0xee);
 961    #endif
 962
 963    (void)linuxkm_op_test_wrapper();
 964
 965    #ifdef WC_LINUXKM_HAVE_STACK_DEBUG
 966        stack_usage = wc_linuxkm_stack_hwm_measure_rel(0xee);
 967        pr_info("STACK INFO: rel usage by linuxkm_op_test_wrapper(): %lu\n", stack_usage);
 968        /* shush up false stack HWM reading by kernel: */
 969        wc_linuxkm_stack_hwm_prepare(0);
 970    }
 971    #endif
 972
 973    WOLFSSL_ATOMIC_STORE(*conTestFailure_ptr, 0);
 974    {
 975        int i;
 976        for (i = 0; i < FIPS_CAST_COUNT; ++i)
 977            fipsCastStatus_put(i, FIPS_CAST_STATE_INIT);
 978    }
 979    /* note, must call fipsEntry() here, not wolfCrypt_IntegrityTest_fips(),
 980     * because wc_GetCastStatus_fips(FIPS_CAST_HMAC_SHA2_256) isn't available
 981     * anymore.
 982     */
 983    if (WC_SIG_IGNORE_BEGIN() >= 0) {
 984        fipsEntry();
 985        (void)WC_SIG_IGNORE_END();
 986    }
 987    else
 988        pr_err("ERROR: WC_SIG_IGNORE_BEGIN() failed.\n");
 989    ret = wolfCrypt_GetStatus_fips();
 990    if (ret != 0) {
 991        pr_err("ERROR: wolfCrypt_GetStatus_fips() after reset failed with code %d: %s\n", ret, wc_GetErrorString(ret));
 992        return -ECANCELED;
 993    }
 994#endif
 995#endif /* FIPS_OPTEST */
 996
 997#ifndef NO_CRYPT_TEST
 998    #ifdef WC_LINUXKM_HAVE_STACK_DEBUG
 999    {
1000        unsigned long stack_usage;
1001        stack_usage = wc_linuxkm_stack_current();
1002        pr_info("STACK INFO: usage at call to wolfcrypt_test(): %lu of %lu total\n", stack_usage, THREAD_SIZE);
1003        wc_linuxkm_stack_hwm_prepare(0xee);
1004    #endif
1005        ret = wolfcrypt_test(NULL);
1006    #ifdef WC_LINUXKM_HAVE_STACK_DEBUG
1007        stack_usage = wc_linuxkm_stack_hwm_measure_rel(0xee);
1008        pr_info("STACK INFO: rel usage by wolfcrypt_test(): %lu\n", stack_usage);
1009        /* shush up false stack HWM reading by kernel: */
1010        wc_linuxkm_stack_hwm_prepare(0);
1011    }
1012    #endif
1013    if (ret < 0) {
1014        pr_err("ERROR: wolfcrypt self-test failed with return code %d.\n", ret);
1015        (void)libwolfssl_cleanup();
1016        msleep(10);
1017        return -ECANCELED;
1018    }
1019    pr_info("wolfCrypt self-test passed.\n");
1020#else
1021#if !defined(HAVE_FIPS) || FIPS_VERSION3_LE(5,2,0)
1022    pr_info("skipping full wolfcrypt_test() "
1023            "(configure with --enable-crypttests to enable).\n");
1024#endif
1025#endif
1026
1027#ifdef LINUXKM_LKCAPI_REGISTER
1028#ifdef LINUXKM_LKCAPI_REGISTER_ONLY_ON_COMMAND
1029    ret = linuxkm_lkcapi_sysfs_install();
1030
1031    if (ret) {
1032        pr_err("ERROR: linuxkm_lkcapi_sysfs_install() failed with return code %d.\n", ret);
1033        (void)libwolfssl_cleanup();
1034        msleep(10);
1035        return -ECANCELED;
1036    }
1037#else /* !LINUXKM_LKCAPI_REGISTER_ONLY_ON_COMMAND */
1038    #ifdef WC_LINUXKM_HAVE_STACK_DEBUG
1039    {
1040        unsigned long stack_usage;
1041        stack_usage = wc_linuxkm_stack_current();
1042        pr_info("STACK INFO: usage at call to linuxkm_lkcapi_register(): %lu of %lu total\n", stack_usage, THREAD_SIZE);
1043        wc_linuxkm_stack_hwm_prepare(0xee);
1044    #endif
1045        ret = linuxkm_lkcapi_register();
1046    #ifdef WC_LINUXKM_HAVE_STACK_DEBUG
1047        stack_usage = wc_linuxkm_stack_hwm_measure_rel(0xee);
1048        pr_info("STACK INFO: rel usage by linuxkm_lkcapi_register(): %lu\n", stack_usage);
1049        /* shush up false stack HWM reading by kernel: */
1050        wc_linuxkm_stack_hwm_prepare(0);
1051    }
1052    #endif
1053
1054    if (ret) {
1055        pr_err("ERROR: linuxkm_lkcapi_register() failed with return code %d.\n", ret);
1056        linuxkm_lkcapi_unregister();
1057        (void)libwolfssl_cleanup();
1058        msleep(10);
1059        return -ECANCELED;
1060    }
1061#endif /* !LINUXKM_LKCAPI_REGISTER_ONLY_ON_COMMAND */
1062#endif /* LINUXKM_LKCAPI_REGISTER */
1063
1064#ifdef HAVE_FIPS
1065    (void)linuxkm_lkcapi_sysfs_install_node(&FIPS_rerun_self_test_attr, &installed_sysfs_FIPS_files);
1066#endif
1067
1068#ifdef WOLFSSL_LINUXKM_BENCHMARKS
1069    #ifdef WC_LINUXKM_HAVE_STACK_DEBUG
1070    {
1071        unsigned long stack_usage;
1072        stack_usage = wc_linuxkm_stack_current();
1073        pr_info("STACK INFO: usage at call to wolfcrypt_benchmark_main(): %lu of %lu total\n", stack_usage, THREAD_SIZE);
1074        wc_linuxkm_stack_hwm_prepare(0xee);
1075    #endif
1076        wolfcrypt_benchmark_main(0, (char**)NULL);
1077    #ifdef WC_LINUXKM_HAVE_STACK_DEBUG
1078        stack_usage = wc_linuxkm_stack_hwm_measure_rel(0xee);
1079        pr_info("STACK INFO: rel usage by wolfcrypt_benchmark_main(): %lu\n", stack_usage);
1080        /* shush up false stack HWM reading by kernel: */
1081        wc_linuxkm_stack_hwm_prepare(0);
1082    }
1083    #endif
1084#endif
1085
1086#ifdef WOLFCRYPT_ONLY
1087    pr_info("wolfCrypt " LIBWOLFSSL_VERSION_STRING " loaded%s"
1088            ".\nSee https://www.wolfssl.com/ for more information.\n"
1089            "wolfCrypt Copyright (C) 2006-present wolfSSL Inc.  Licensed under " WOLFSSL_LICENSE ".\n",
1090#ifdef CONFIG_MODULE_SIG
1091            THIS_MODULE->sig_ok ? " with valid module signature" : " without valid module signature"
1092#else
1093            ""
1094#endif
1095        );
1096#else
1097    pr_info("wolfSSL " LIBWOLFSSL_VERSION_STRING " loaded%s"
1098            ".\nSee https://www.wolfssl.com/ for more information.\n"
1099            "wolfSSL Copyright (C) 2006-present wolfSSL Inc.  Licensed under " WOLFSSL_LICENSE ".\n",
1100#ifdef CONFIG_MODULE_SIG
1101            THIS_MODULE->sig_ok ? " with valid module signature" : " without valid module signature"
1102#else
1103            ""
1104#endif
1105        );
1106#endif
1107
1108    return 0;
1109}
1110
1111module_init(wolfssl_init);
1112
1113#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 0, 0)
1114static void __exit wolfssl_exit(void)
1115#else
1116static void wolfssl_exit(void)
1117#endif
1118{
1119#ifdef HAVE_FIPS
1120    int ret;
1121
1122    (void)linuxkm_lkcapi_sysfs_deinstall_node(&FIPS_rerun_self_test_attr, &installed_sysfs_FIPS_files);
1123#ifdef FIPS_OPTEST
1124    (void)linuxkm_lkcapi_sysfs_deinstall_node(&FIPS_optest_trig_attr, &installed_sysfs_FIPS_optest_trig_files);
1125#endif
1126#endif
1127
1128#ifdef LINUXKM_LKCAPI_REGISTER
1129    (void)linuxkm_lkcapi_unregister();
1130    (void)linuxkm_lkcapi_sysfs_deinstall();
1131#endif
1132
1133#ifdef HAVE_FIPS
1134    ret = wc_RunAllCast_fips();
1135    if (ret != 0) {
1136        pr_err("ERROR: wc_RunAllCast_fips() failed at shutdown with return value %d\n", ret);
1137    }
1138    else
1139        pr_info("wolfCrypt FIPS re-self-test succeeded at unload: all algorithms re-verified.\n");
1140#endif
1141
1142    (void)libwolfssl_cleanup();
1143
1144    return;
1145}
1146
1147module_exit(wolfssl_exit);
1148
1149#if defined(LINUXKM_LKCAPI_REGISTER) || !defined(WOLFSSL_NO_ASM)
1150    /* When registering algorithms with crypto_register_skcipher() and friends,
1151     * or using kernel_fpu_begin_mask() and _end() to wrap vector register
1152     * usage, we use a "GPL" license unconditionally here to meet the GPL-only
1153     * requirements for those calls, satisfying license_is_gpl_compatible() (see
1154     * /usr/src/linux/include/linux/license.h).
1155     */
1156    MODULE_LICENSE("GPL");
1157#else
1158    MODULE_LICENSE(WOLFSSL_LICENSE);
1159#endif
1160
1161MODULE_AUTHOR("https://www.wolfssl.com/");
1162MODULE_DESCRIPTION("libwolfssl cryptographic and protocol facilities");
1163MODULE_VERSION(LIBWOLFSSL_VERSION_STRING);
1164
1165#ifdef WC_SYM_RELOC_TABLES
1166
1167extern const struct wc_reloc_table_ent wc_linuxkm_pie_text_reloc_tab[];
1168extern const unsigned int wc_linuxkm_pie_text_reloc_tab_length;
1169extern const struct wc_reloc_table_ent wc_linuxkm_pie_rodata_reloc_tab[];
1170extern const unsigned int wc_linuxkm_pie_rodata_reloc_tab_length;
1171
1172static const struct wc_reloc_table_segments seg_map = {
1173    .start = 0, .end = 0,
1174    .text_start = (size_t)(uintptr_t)__wc_text_start,
1175    .text_end = (size_t)(uintptr_t)__wc_text_end,
1176    .text_reloc_tab.start = (size_t)(uintptr_t)wc_linuxkm_pie_text_reloc_tab,
1177    .text_reloc_tab.end = 0,
1178    .text_reloc_tab.len_start = (size_t)(uintptr_t)&wc_linuxkm_pie_text_reloc_tab_length,
1179    .text_reloc_tab.len_end = 0,
1180    .rodata_reloc_tab.start = (size_t)(uintptr_t)wc_linuxkm_pie_rodata_reloc_tab,
1181    .rodata_reloc_tab.end = 0,
1182    .rodata_reloc_tab.len_start = (size_t)(uintptr_t)&wc_linuxkm_pie_rodata_reloc_tab_length,
1183    .rodata_reloc_tab.len_end = 0,
1184#ifdef HAVE_FIPS
1185#ifdef WC_USE_PIE_FENCEPOSTS_FOR_FIPS
1186    .fips_text_start = (size_t)(uintptr_t)__wc_text_start,
1187    .fips_text_end = (size_t)(uintptr_t)__wc_text_end,
1188#else
1189    .fips_text_start = (size_t)(uintptr_t)wolfCrypt_FIPS_first,
1190    .fips_text_end = (size_t)(uintptr_t)wolfCrypt_FIPS_last,
1191#endif
1192#endif /* HAVE_FIPS */
1193    .rodata_start = (size_t)(uintptr_t)__wc_rodata_start,
1194    .rodata_end = (size_t)(uintptr_t)__wc_rodata_end,
1195#ifdef HAVE_FIPS
1196#ifdef WC_USE_PIE_FENCEPOSTS_FOR_FIPS
1197    .fips_rodata_start = (size_t)(uintptr_t)__wc_rodata_start,
1198    .fips_rodata_end = (size_t)(uintptr_t)__wc_rodata_end,
1199#else
1200    .fips_rodata_start = (size_t)(uintptr_t)wolfCrypt_FIPS_ro_start,
1201    .fips_rodata_end = (size_t)(uintptr_t)wolfCrypt_FIPS_ro_end,
1202#endif
1203    #if FIPS_VERSION3_GE(6,0,0) || defined(WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE)
1204    .verifyCore_start = (uintptr_t)verifyCore,
1205    .verifyCore_end = (uintptr_t)verifyCore + FIPS_IN_CORE_DIGEST_SIZE*2 + 1,
1206    #endif
1207#endif /* HAVE_FIPS */
1208    .data_start = (size_t)(uintptr_t)__wc_rwdata_start,
1209    .data_end = (size_t)(uintptr_t)__wc_rwdata_end,
1210    .bss_start = (size_t)(uintptr_t)__wc_bss_start,
1211    .bss_end = (size_t)(uintptr_t)__wc_bss_end,
1212    .text_is_live = 1
1213};
1214
1215ssize_t wc_linuxkm_normalize_relocations(
1216    const u8 *seg_in,
1217    size_t *seg_in_out_len,
1218    u8 *seg_out,
1219    ssize_t *cur_index_p)
1220{
1221    return wc_reloc_normalize_segment(seg_in, seg_in_out_len, seg_out, cur_index_p, &seg_map,
1222#ifdef DEBUG_LINUXKM_PIE_SUPPORT
1223                                   &reloc_counts
1224#else
1225                                   NULL
1226#endif
1227        );
1228}
1229
1230ssize_t wc_linuxkm_normalize_relocations_noresize(
1231    const u8 *seg_in,
1232    size_t seg_in_len,
1233    u8 *seg_out,
1234    ssize_t *cur_index_p)
1235{
1236    ssize_t ret;
1237    ret = wc_reloc_normalize_segment(seg_in, &seg_in_len, seg_out, cur_index_p, &seg_map,
1238#ifdef DEBUG_LINUXKM_PIE_SUPPORT
1239                                   &reloc_counts
1240#else
1241                                   NULL
1242#endif
1243        );
1244    if (ret < 0)
1245        return ret;
1246    if ((size_t)ret != seg_in_len)
1247        return -EINVAL;
1248    else
1249        return seg_in_len;
1250}
1251
1252#elif defined(HAVE_FIPS)
1253
1254static const struct wc_reloc_table_segments seg_map = {
1255    .start = 0, .end = 0,
1256#ifdef WC_USE_PIE_FENCEPOSTS_FOR_FIPS
1257    .fips_text_start = (size_t)(uintptr_t)__wc_text_start,
1258    .fips_text_end = (size_t)(uintptr_t)__wc_text_end,
1259#else
1260    .fips_text_start = (size_t)(uintptr_t)wolfCrypt_FIPS_first,
1261    .fips_text_end = (size_t)(uintptr_t)wolfCrypt_FIPS_last,
1262#endif
1263#ifdef WC_USE_PIE_FENCEPOSTS_FOR_FIPS
1264    .fips_rodata_start = (size_t)(uintptr_t)__wc_rodata_start,
1265    .fips_rodata_end = (size_t)(uintptr_t)__wc_rodata_end,
1266#else
1267    .fips_rodata_start = (size_t)(uintptr_t)wolfCrypt_FIPS_ro_start,
1268    .fips_rodata_end = (size_t)(uintptr_t)wolfCrypt_FIPS_ro_end,
1269#endif
1270    #if FIPS_VERSION3_GE(6,0,0) || defined(WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE)
1271    .verifyCore_start = (uintptr_t)verifyCore,
1272    .verifyCore_end = (uintptr_t)verifyCore + FIPS_IN_CORE_DIGEST_SIZE*2 + 1
1273    #endif
1274};
1275
1276#endif /* !WC_SYM_RELOC_TABLES && HAVE_FIPS */
1277
1278#ifdef WC_SYM_RELOC_TABLES
1279
1280/* get_current() is an inline or macro, depending on the target -- sidestep the
1281 * whole issue with a wrapper func.
1282 */
1283static struct task_struct *my_get_current_thread(void) {
1284    return get_current();
1285}
1286
1287/* preempt_count() is an inline function in arch/x86/include/asm/preempt.h that
1288 * accesses __preempt_count, which is an int array declared with
1289 * DECLARE_PER_CPU_CACHE_HOT.
1290 */
1291static int my_preempt_count(void) {
1292    return preempt_count();
1293}
1294
1295static int set_up_wolfssl_linuxkm_pie_redirect_table(void) {
1296    memset(
1297        &wolfssl_linuxkm_pie_redirect_table,
1298        0,
1299        sizeof wolfssl_linuxkm_pie_redirect_table);
1300
1301#ifdef HAVE_FIPS
1302    wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_normalize_relocations =
1303        wc_linuxkm_normalize_relocations;
1304    wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_normalize_relocations_noresize =
1305        wc_linuxkm_normalize_relocations_noresize;
1306#endif
1307
1308#ifndef __ARCH_MEMCMP_NO_REDIRECT
1309    wolfssl_linuxkm_pie_redirect_table.memcmp = memcmp;
1310#endif
1311#ifndef CONFIG_FORTIFY_SOURCE
1312#ifndef __ARCH_MEMCPY_NO_REDIRECT
1313    wolfssl_linuxkm_pie_redirect_table.memcpy = memcpy;
1314#endif
1315#ifndef __ARCH_MEMSET_NO_REDIRECT
1316    wolfssl_linuxkm_pie_redirect_table.memset = memset;
1317#endif
1318#ifndef __ARCH_MEMMOVE_NO_REDIRECT
1319    wolfssl_linuxkm_pie_redirect_table.memmove = memmove;
1320#endif
1321#endif /* !CONFIG_FORTIFY_SOURCE */
1322#ifndef __ARCH_STRCMP_NO_REDIRECT
1323    wolfssl_linuxkm_pie_redirect_table.strcmp = strcmp;
1324#endif
1325#ifndef __ARCH_STRNCMP_NO_REDIRECT
1326    wolfssl_linuxkm_pie_redirect_table.strncmp = strncmp;
1327#endif
1328#ifndef __ARCH_STRCASECMP_NO_REDIRECT
1329    wolfssl_linuxkm_pie_redirect_table.strcasecmp = strcasecmp;
1330#endif
1331#ifndef __ARCH_STRNCASECMP_NO_REDIRECT
1332    wolfssl_linuxkm_pie_redirect_table.strncasecmp = strncasecmp;
1333#endif
1334#ifndef __ARCH_STRLEN_NO_REDIRECT
1335    wolfssl_linuxkm_pie_redirect_table.strlen = strlen;
1336#endif
1337#ifndef __ARCH_STRSTR_NO_REDIRECT
1338    wolfssl_linuxkm_pie_redirect_table.strstr = strstr;
1339#endif
1340#ifndef __ARCH_STRNCPY_NO_REDIRECT
1341    wolfssl_linuxkm_pie_redirect_table.strncpy = strncpy;
1342#endif
1343#ifndef __ARCH_STRNCAT_NO_REDIRECT
1344    wolfssl_linuxkm_pie_redirect_table.strncat = strncat;
1345#endif
1346    wolfssl_linuxkm_pie_redirect_table.kstrtoll = kstrtoll;
1347
1348    #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0)) || \
1349        (defined(RHEL_MAJOR) && \
1350         ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
1351        wolfssl_linuxkm_pie_redirect_table._printk = _printk;
1352    #else
1353        wolfssl_linuxkm_pie_redirect_table.printk = printk;
1354    #endif
1355
1356#ifdef CONFIG_FORTIFY_SOURCE
1357    wolfssl_linuxkm_pie_redirect_table.__warn_printk = __warn_printk;
1358#endif
1359
1360    wolfssl_linuxkm_pie_redirect_table.snprintf = snprintf;
1361
1362    wolfssl_linuxkm_pie_redirect_table._ctype = _ctype;
1363
1364#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0)
1365    wolfssl_linuxkm_pie_redirect_table.kmalloc_noprof = kmalloc_noprof;
1366    wolfssl_linuxkm_pie_redirect_table.krealloc_node_align_noprof = krealloc_node_align_noprof;
1367    wolfssl_linuxkm_pie_redirect_table.kzalloc_noprof = kzalloc_noprof;
1368    wolfssl_linuxkm_pie_redirect_table.__kvmalloc_node_noprof = __kvmalloc_node_noprof;
1369    wolfssl_linuxkm_pie_redirect_table.__kmalloc_cache_noprof = __kmalloc_cache_noprof;
1370#ifdef HAVE_KVREALLOC
1371    wolfssl_linuxkm_pie_redirect_table.kvrealloc_node_align_noprof = kvrealloc_node_align_noprof;
1372#endif
1373#elif LINUX_VERSION_CODE >= KERNEL_VERSION(6, 11, 0)
1374    wolfssl_linuxkm_pie_redirect_table.kmalloc_noprof = kmalloc_noprof;
1375    wolfssl_linuxkm_pie_redirect_table.krealloc_noprof = krealloc_noprof;
1376    wolfssl_linuxkm_pie_redirect_table.kzalloc_noprof = kzalloc_noprof;
1377    wolfssl_linuxkm_pie_redirect_table.__kvmalloc_node_noprof = __kvmalloc_node_noprof;
1378    wolfssl_linuxkm_pie_redirect_table.__kmalloc_cache_noprof = __kmalloc_cache_noprof;
1379#ifdef HAVE_KVREALLOC
1380    wolfssl_linuxkm_pie_redirect_table.kvrealloc_noprof = kvrealloc_noprof;
1381#endif
1382#elif LINUX_VERSION_CODE >= KERNEL_VERSION(6, 10, 0)
1383    wolfssl_linuxkm_pie_redirect_table.kmalloc_noprof = kmalloc_noprof;
1384    wolfssl_linuxkm_pie_redirect_table.krealloc_noprof = krealloc_noprof;
1385    wolfssl_linuxkm_pie_redirect_table.kzalloc_noprof = kzalloc_noprof;
1386    wolfssl_linuxkm_pie_redirect_table.kvmalloc_node_noprof = kvmalloc_node_noprof;
1387    wolfssl_linuxkm_pie_redirect_table.kmalloc_trace_noprof = kmalloc_trace_noprof;
1388#ifdef HAVE_KVREALLOC
1389    wolfssl_linuxkm_pie_redirect_table.kvrealloc_noprof = kvrealloc_noprof;
1390#endif
1391#else
1392    wolfssl_linuxkm_pie_redirect_table.kmalloc = kmalloc;
1393    wolfssl_linuxkm_pie_redirect_table.krealloc = krealloc;
1394#ifdef HAVE_KVMALLOC
1395    wolfssl_linuxkm_pie_redirect_table.kvmalloc_node = kvmalloc_node;
1396#endif
1397#ifdef HAVE_KVREALLOC
1398    wolfssl_linuxkm_pie_redirect_table.kvrealloc = kvrealloc;
1399#endif
1400    #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0) || \
1401        (defined(RHEL_MAJOR) &&                                    \
1402         ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
1403        wolfssl_linuxkm_pie_redirect_table.kmalloc_trace =
1404            kmalloc_trace;
1405    #else
1406        wolfssl_linuxkm_pie_redirect_table.kmem_cache_alloc_trace =
1407            kmem_cache_alloc_trace;
1408        wolfssl_linuxkm_pie_redirect_table.kmalloc_order_trace =
1409            kmalloc_order_trace;
1410    #endif
1411#endif
1412
1413    wolfssl_linuxkm_pie_redirect_table.kfree = kfree;
1414    wolfssl_linuxkm_pie_redirect_table.ksize = ksize;
1415#ifdef HAVE_KVMALLOC
1416    wolfssl_linuxkm_pie_redirect_table.kvfree = kvfree;
1417#endif
1418
1419#ifndef LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT
1420    wolfssl_linuxkm_pie_redirect_table.get_random_bytes = get_random_bytes;
1421#endif
1422    #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
1423        wolfssl_linuxkm_pie_redirect_table.getnstimeofday =
1424            getnstimeofday;
1425    #elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
1426        wolfssl_linuxkm_pie_redirect_table.current_kernel_time64 =
1427            current_kernel_time64;
1428    #else
1429        wolfssl_linuxkm_pie_redirect_table.ktime_get_coarse_real_ts64 =
1430            ktime_get_coarse_real_ts64;
1431    #endif
1432
1433    wolfssl_linuxkm_pie_redirect_table.get_current = my_get_current_thread;
1434
1435#if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86)
1436    wolfssl_linuxkm_pie_redirect_table.allocate_wolfcrypt_linuxkm_fpu_states = allocate_wolfcrypt_linuxkm_fpu_states;
1437    wolfssl_linuxkm_pie_redirect_table.wc_can_save_vector_registers_x86 = wc_can_save_vector_registers_x86;
1438    wolfssl_linuxkm_pie_redirect_table.free_wolfcrypt_linuxkm_fpu_states = free_wolfcrypt_linuxkm_fpu_states;
1439    wolfssl_linuxkm_pie_redirect_table.wc_restore_vector_registers_x86 = wc_restore_vector_registers_x86;
1440    wolfssl_linuxkm_pie_redirect_table.wc_save_vector_registers_x86 = wc_save_vector_registers_x86;
1441#elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)
1442    #error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unsupported architecture.
1443#endif /* WOLFSSL_USE_SAVE_VECTOR_REGISTERS */
1444
1445    #ifndef CONFIG_PREEMPT_RT
1446        wolfssl_linuxkm_pie_redirect_table.__mutex_init = __mutex_init;
1447    #else
1448        wolfssl_linuxkm_pie_redirect_table.__rt_mutex_init = __rt_mutex_init;
1449        wolfssl_linuxkm_pie_redirect_table.rt_mutex_base_init = rt_mutex_base_init;
1450        wolfssl_linuxkm_pie_redirect_table.rt_spin_lock = rt_spin_lock;
1451        wolfssl_linuxkm_pie_redirect_table.rt_spin_unlock = rt_spin_unlock;
1452    #endif
1453    #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
1454        wolfssl_linuxkm_pie_redirect_table.mutex_lock_nested = mutex_lock_nested;
1455    #else
1456        wolfssl_linuxkm_pie_redirect_table.mutex_lock = mutex_lock;
1457    #endif
1458    wolfssl_linuxkm_pie_redirect_table.mutex_unlock = mutex_unlock;
1459    #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
1460        wolfssl_linuxkm_pie_redirect_table.mutex_destroy = mutex_destroy;
1461    #endif
1462
1463#ifdef HAVE_FIPS
1464
1465#ifdef WC_USE_PIE_FENCEPOSTS_FOR_FIPS
1466    /* use __wc_text_start and __wc_text_end, not wolfCrypt_FIPS_first and
1467     * wolfCrypt_FIPS_last, thereby including the whole container in the HMAC
1468     * span.  Note there are runtime asserts at entry to wolfssl_init() above
1469     * confirming that __wc_*_{start,end} correctly contain the wolfCrypt_FIPS_*
1470     * fenceposts.
1471     */
1472    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_first =
1473        __wc_text_start;
1474    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_last =
1475        __wc_text_end;
1476    /* ditto for wolfCrypt_FIPS_ro_start and wolfCrypt_FIPS_ro_end. */
1477    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ro_start =
1478        &__wc_rodata_start;
1479    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ro_end =
1480        &__wc_rodata_end;
1481#else
1482    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_first =
1483        wolfCrypt_FIPS_first;
1484    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_last =
1485        wolfCrypt_FIPS_last;
1486    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ro_start =
1487        &wolfCrypt_FIPS_ro_start;
1488    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ro_end =
1489        &wolfCrypt_FIPS_ro_end;
1490#endif
1491
1492    #if FIPS_VERSION3_GE(6,0,0)
1493#ifndef NO_AES
1494    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_AES_sanity =
1495        wolfCrypt_FIPS_AES_sanity;
1496    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_aes_ro_sanity =
1497        &wolfCrypt_FIPS_aes_ro_sanity;
1498#if defined(WOLFSSL_CMAC) && defined(WOLFSSL_AES_DIRECT)
1499    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_CMAC_sanity =
1500        wolfCrypt_FIPS_CMAC_sanity;
1501    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_cmac_ro_sanity =
1502        &wolfCrypt_FIPS_cmac_ro_sanity;
1503#endif
1504#endif
1505#ifndef NO_DH
1506    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_DH_sanity =
1507        wolfCrypt_FIPS_DH_sanity;
1508    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_dh_ro_sanity =
1509        &wolfCrypt_FIPS_dh_ro_sanity;
1510#endif
1511#ifdef HAVE_ECC
1512    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ECC_sanity =
1513        wolfCrypt_FIPS_ECC_sanity;
1514    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ecc_ro_sanity =
1515        &wolfCrypt_FIPS_ecc_ro_sanity;
1516#endif
1517#ifdef HAVE_ED25519
1518    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ED25519_sanity =
1519        wolfCrypt_FIPS_ED25519_sanity;
1520    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ed25519_ro_sanity =
1521        &wolfCrypt_FIPS_ed25519_ro_sanity;
1522#endif
1523#ifdef HAVE_ED448
1524    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ED448_sanity =
1525        wolfCrypt_FIPS_ED448_sanity;
1526    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ed448_ro_sanity =
1527        &wolfCrypt_FIPS_ed448_ro_sanity;
1528#endif
1529    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_HMAC_sanity =
1530        wolfCrypt_FIPS_HMAC_sanity;
1531    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_hmac_ro_sanity =
1532        &wolfCrypt_FIPS_hmac_ro_sanity;
1533#ifndef NO_KDF
1534    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_KDF_sanity =
1535        wolfCrypt_FIPS_KDF_sanity;
1536    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_kdf_ro_sanity =
1537        &wolfCrypt_FIPS_kdf_ro_sanity;
1538#endif
1539#ifdef HAVE_PBKDF2
1540    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_PBKDF_sanity =
1541        wolfCrypt_FIPS_PBKDF_sanity;
1542    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_pbkdf_ro_sanity =
1543        &wolfCrypt_FIPS_pbkdf_ro_sanity;
1544#endif
1545#ifdef HAVE_HASHDRBG
1546    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_DRBG_sanity =
1547        wolfCrypt_FIPS_DRBG_sanity;
1548    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_drbg_ro_sanity =
1549        &wolfCrypt_FIPS_drbg_ro_sanity;
1550#endif
1551#ifndef NO_RSA
1552    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_RSA_sanity =
1553        wolfCrypt_FIPS_RSA_sanity;
1554    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_rsa_ro_sanity =
1555        &wolfCrypt_FIPS_rsa_ro_sanity;
1556#endif
1557#ifndef NO_SHA
1558    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_SHA_sanity =
1559        wolfCrypt_FIPS_SHA_sanity;
1560    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_sha_ro_sanity =
1561        &wolfCrypt_FIPS_sha_ro_sanity;
1562#endif
1563#ifndef NO_SHA256
1564    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_SHA256_sanity =
1565        wolfCrypt_FIPS_SHA256_sanity;
1566    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_sha256_ro_sanity =
1567        &wolfCrypt_FIPS_sha256_ro_sanity;
1568#endif
1569#ifdef WOLFSSL_SHA512
1570    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_SHA512_sanity =
1571        wolfCrypt_FIPS_SHA512_sanity;
1572    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_sha512_ro_sanity =
1573        &wolfCrypt_FIPS_sha512_ro_sanity;
1574#endif
1575#ifdef WOLFSSL_SHA3
1576    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_SHA3_sanity =
1577        wolfCrypt_FIPS_SHA3_sanity;
1578    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_sha3_ro_sanity =
1579        &wolfCrypt_FIPS_sha3_ro_sanity;
1580#endif
1581    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_FT_sanity =
1582        wolfCrypt_FIPS_FT_sanity;
1583    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_ft_ro_sanity =
1584        &wolfCrypt_FIPS_ft_ro_sanity;
1585    wolfssl_linuxkm_pie_redirect_table.wolfCrypt_FIPS_f_ro_sanity =
1586        &wolfCrypt_FIPS_f_ro_sanity;
1587    wolfssl_linuxkm_pie_redirect_table.wc_RunAllCast_fips =
1588        wc_RunAllCast_fips;
1589    #endif
1590#endif
1591
1592#if !defined(WOLFCRYPT_ONLY) && !defined(NO_CERTS)
1593    wolfssl_linuxkm_pie_redirect_table.GetCA = GetCA;
1594#ifndef NO_SKID
1595    wolfssl_linuxkm_pie_redirect_table.GetCAByName = GetCAByName;
1596#ifdef HAVE_OCSP
1597    wolfssl_linuxkm_pie_redirect_table.GetCAByKeyHash = GetCAByKeyHash;
1598#endif /* HAVE_OCSP */
1599#endif /* NO_SKID */
1600#ifdef WOLFSSL_AKID_NAME
1601    wolfssl_linuxkm_pie_redirect_table.GetCAByAKID = GetCAByAKID;
1602#endif /* WOLFSSL_AKID_NAME */
1603#if defined(OPENSSL_EXTRA) || defined(OPENSSL_EXTRA_X509_SMALL)
1604    wolfssl_linuxkm_pie_redirect_table.wolfSSL_X509_NAME_add_entry_by_NID = wolfSSL_X509_NAME_add_entry_by_NID;
1605    wolfssl_linuxkm_pie_redirect_table.wolfSSL_X509_NAME_free = wolfSSL_X509_NAME_free;
1606    wolfssl_linuxkm_pie_redirect_table.wolfSSL_X509_NAME_new_ex = wolfSSL_X509_NAME_new_ex;
1607#endif /* OPENSSL_EXTRA || OPENSSL_EXTRA_X509_SMALL */
1608#endif /* !WOLFCRYPT_ONLY && !NO_CERTS */
1609
1610    wolfssl_linuxkm_pie_redirect_table.dump_stack = dump_stack;
1611
1612    wolfssl_linuxkm_pie_redirect_table.preempt_count = my_preempt_count;
1613#ifndef _raw_spin_lock_irqsave
1614    wolfssl_linuxkm_pie_redirect_table._raw_spin_lock_irqsave = _raw_spin_lock_irqsave;
1615#endif
1616#ifndef _raw_spin_trylock
1617    wolfssl_linuxkm_pie_redirect_table._raw_spin_trylock = _raw_spin_trylock;
1618#endif
1619#ifndef _raw_spin_unlock_irqrestore
1620    wolfssl_linuxkm_pie_redirect_table._raw_spin_unlock_irqrestore = _raw_spin_unlock_irqrestore;
1621#endif
1622    wolfssl_linuxkm_pie_redirect_table._cond_resched = _cond_resched;
1623
1624#ifndef WOLFSSL_LINUXKM_USE_MUTEXES
1625    wolfssl_linuxkm_pie_redirect_table.wc_lkm_LockMutex = wc_lkm_LockMutex;
1626#endif
1627
1628#ifdef CONFIG_ARM64
1629#ifndef CONFIG_ARCH_TEGRA
1630#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)
1631    wolfssl_linuxkm_pie_redirect_table.alt_cb_patch_nops = alt_cb_patch_nops;
1632#endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0) */
1633    wolfssl_linuxkm_pie_redirect_table.queued_spin_lock_slowpath = queued_spin_lock_slowpath;
1634#endif
1635#endif
1636
1637    wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_sig_ignore_begin = wc_linuxkm_sig_ignore_begin;
1638    wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_sig_ignore_end = wc_linuxkm_sig_ignore_end;
1639    wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_check_for_intr_signals = wc_linuxkm_check_for_intr_signals;
1640    wolfssl_linuxkm_pie_redirect_table.wc_linuxkm_relax_long_loop = wc_linuxkm_relax_long_loop;
1641
1642#ifdef CONFIG_KASAN
1643    wolfssl_linuxkm_pie_redirect_table.kasan_disable_current = kasan_disable_current;
1644    wolfssl_linuxkm_pie_redirect_table.kasan_enable_current = kasan_enable_current;
1645#endif
1646
1647    /* runtime assert that the table has no null slots after initialization. */
1648    {
1649        unsigned long *i;
1650        static_assert(sizeof(unsigned long) == sizeof(void *),
1651                      "unexpected pointer size");
1652        for (i = (unsigned long *)&wolfssl_linuxkm_pie_redirect_table;
1653             i < (unsigned long *)&wolfssl_linuxkm_pie_redirect_table._last_slot;
1654             ++i)
1655            if (*i == 0) {
1656                pr_err("ERROR: wolfCrypt container redirect table initialization was "
1657                       "incomplete [%u].\n",
1658                       (unsigned)(i-(unsigned long *)&wolfssl_linuxkm_pie_redirect_table));
1659                return -EFAULT;
1660            }
1661    }
1662
1663    return 0;
1664}
1665
1666#endif /* WC_SYM_RELOC_TABLES */
1667
1668#if defined(HAVE_FIPS) && defined(WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE)
1669
1670#include <wolfssl/wolfcrypt/coding.h>
1671
1672PRAGMA_GCC_DIAG_PUSH
1673PRAGMA_GCC("GCC diagnostic ignored \"-Wnested-externs\"")
1674PRAGMA_GCC("GCC diagnostic ignored \"-Wpointer-arith\"")
1675PRAGMA_GCC("GCC diagnostic ignored \"-Wunused-parameter\"")
1676#include <crypto/hash.h>
1677PRAGMA_GCC_DIAG_POP
1678
1679/* failsafe definitions for FIPS <5.3 */
1680#ifndef FIPS_IN_CORE_DIGEST_SIZE
1681    #ifndef NO_SHA256
1682        #define FIPS_IN_CORE_DIGEST_SIZE 32
1683        #define FIPS_IN_CORE_HASH_TYPE   WC_SHA256
1684        #define FIPS_IN_CORE_KEY_SZ      32
1685        #define FIPS_IN_CORE_VERIFY_SZ   FIPS_IN_CORE_KEY_SZ
1686    #elif defined(WOLFSSL_SHA384)
1687        #define FIPS_IN_CORE_DIGEST_SIZE 48
1688        #define FIPS_IN_CORE_HASH_TYPE   WC_SHA384
1689        #define FIPS_IN_CORE_KEY_SZ      48
1690        #define FIPS_IN_CORE_VERIFY_SZ   FIPS_IN_CORE_KEY_SZ
1691    #else
1692        #error Unsupported FIPS hash alg.
1693    #endif
1694#endif
1695
1696extern const char coreKey[FIPS_IN_CORE_KEY_SZ*2 + 1];
1697extern const unsigned int wolfCrypt_FIPS_ro_start[];
1698extern const unsigned int wolfCrypt_FIPS_ro_end[];
1699
1700static int linux_fips_hmac_setkey(struct shash_desc *desc, const byte *key, word32 key_len) {
1701    int ret = crypto_shash_setkey(desc->tfm, key, key_len);
1702    if (ret) {
1703        pr_err("ERROR: crypto_shash_setkey failed: err %d\n", ret);
1704        return BAD_STATE_E;
1705    }
1706
1707    ret = crypto_shash_init(desc);
1708    if (ret) {
1709        pr_err("ERROR: crypto_shash_init failed: err %d\n", ret);
1710        return BAD_STATE_E;
1711    }
1712    return 0;
1713}
1714static int linux_fips_hmac_update(struct shash_desc *desc, const byte *in, word32 in_len) {
1715    int ret = crypto_shash_update(desc, in, in_len);
1716    if (ret) {
1717        pr_err("ERROR: crypto_shash_update failed: err %d\n", ret);
1718        return BAD_STATE_E;
1719    }
1720    else
1721        return 0;
1722}
1723static int linux_fips_hmac_final(struct shash_desc *desc, byte *out, word32 out_sz) {
1724    int ret;
1725
1726    (void)out_sz;
1727
1728    ret = crypto_shash_final(desc, out);
1729    if (ret) {
1730        pr_err("ERROR: crypto_shash_final failed: err %d\n", ret);
1731        return BAD_STATE_E;
1732    }
1733    else
1734        return 0;
1735}
1736
1737static int updateFipsHash(void)
1738{
1739    struct crypto_shash *tfm = NULL;
1740    struct shash_desc *desc = NULL;
1741    int ret;
1742    word32 verifyCore_size = XSTRLEN(verifyCore) + 1;
1743    char *old_verifyCore = XMALLOC(verifyCore_size, NULL, DYNAMIC_TYPE_TMP_BUFFER);
1744
1745    if (old_verifyCore == NULL)
1746        return MEMORY_E;
1747    XMEMCPY(old_verifyCore, verifyCore, verifyCore_size);
1748
1749    wc_static_assert(((unsigned)FIPS_IN_CORE_HASH_TYPE == (unsigned)WC_SHA256) ||
1750                     ((unsigned)FIPS_IN_CORE_HASH_TYPE == (unsigned)WC_SHA384) ||
1751                     ((unsigned)FIPS_IN_CORE_HASH_TYPE == (unsigned)WC_SHA512) ||
1752                     ((unsigned)FIPS_IN_CORE_HASH_TYPE == (unsigned)WC_SHA3_256) ||
1753                     ((unsigned)FIPS_IN_CORE_HASH_TYPE == (unsigned)WC_SHA3_384) ||
1754                     ((unsigned)FIPS_IN_CORE_HASH_TYPE == (unsigned)WC_SHA3_512));
1755
1756    switch ((unsigned)FIPS_IN_CORE_HASH_TYPE) {
1757    case (unsigned)WC_SHA256:
1758        tfm = crypto_alloc_shash("hmac(sha256)", 0, 0);
1759        break;
1760    case (unsigned)WC_SHA384:
1761        tfm = crypto_alloc_shash("hmac(sha384)", 0, 0);
1762        break;
1763    case (unsigned)WC_SHA512:
1764        tfm = crypto_alloc_shash("hmac(sha512)", 0, 0);
1765        break;
1766    case (unsigned)WC_SHA3_256:
1767        tfm = crypto_alloc_shash("hmac(sha3-256)", 0, 0);
1768        break;
1769    case (unsigned)WC_SHA3_384:
1770        tfm = crypto_alloc_shash("hmac(sha3-384)", 0, 0);
1771        break;
1772    case (unsigned)WC_SHA3_512:
1773        tfm = crypto_alloc_shash("hmac(sha3-512)", 0, 0);
1774        break;
1775    }
1776
1777    if (IS_ERR(tfm)) {
1778        if (PTR_ERR(tfm) == -ENOMEM) {
1779            pr_err("ERROR: crypto_alloc_shash failed: out of memory\n");
1780            ret = MEMORY_E;
1781        } else if (PTR_ERR(tfm) == -ENOENT) {
1782            pr_err("ERROR: crypto_alloc_shash failed: target kernel is missing algorithm implementation for hash type %u\n", FIPS_IN_CORE_HASH_TYPE);
1783            ret = NOT_COMPILED_IN;
1784        } else {
1785            pr_err("ERROR: crypto_alloc_shash failed with ret %ld\n",PTR_ERR(tfm));
1786            ret = HASH_TYPE_E;
1787        }
1788        tfm = NULL;
1789        goto out;
1790    }
1791
1792    {
1793        size_t desc_size = crypto_shash_descsize(tfm) + sizeof *desc;
1794        desc = XMALLOC(desc_size, NULL, DYNAMIC_TYPE_TMP_BUFFER);
1795        if (desc == NULL) {
1796            pr_err("ERROR: failed allocating desc.\n");
1797            ret = MEMORY_E;
1798            goto out;
1799        }
1800        XMEMSET(desc, 0, desc_size);
1801    }
1802
1803    desc->tfm = tfm;
1804
1805    ret = wc_fips_generate_hash(
1806        &seg_map,
1807        FIPS_IN_CORE_DIGEST_SIZE,
1808        coreKey,
1809        desc,
1810        /* NOLINTBEGIN(clang-diagnostic-cast-function-type-strict) */
1811        (wc_fips_verifyCore_hmac_setkey_fn)linux_fips_hmac_setkey,
1812        (wc_fips_verifyCore_hmac_update_fn)linux_fips_hmac_update,
1813        (wc_fips_verifyCore_hmac_final_fn)linux_fips_hmac_final,
1814        /* NOLINTEND(clang-diagnostic-cast-function-type-strict) */
1815        verifyCore,
1816        &verifyCore_size,
1817#if defined(DEBUG_LINUXKM_PIE_SUPPORT) && defined(WC_SYM_RELOC_TABLES)
1818        &reloc_counts
1819#else
1820        NULL
1821#endif
1822        );
1823
1824    if (ret < 0)
1825        goto out;
1826
1827    if (strcmp(old_verifyCore, verifyCore) == 0) {
1828#if defined(DEBUG_LINUXKM_PIE_SUPPORT) || defined(WOLFSSL_LINUXKM_VERBOSE_DEBUG)
1829        pr_info("updateFipsHash: verifyCore already matches [%s]\n", verifyCore);
1830#else
1831        pr_info("updateFipsHash: verifyCore already matches.\n");
1832#endif
1833    }
1834    else {
1835#if defined(DEBUG_LINUXKM_PIE_SUPPORT) || defined(WOLFSSL_LINUXKM_VERBOSE_DEBUG)
1836        pr_info("updateFipsHash: verifyCore updated [%s].\n", verifyCore);
1837#else
1838        pr_info("updateFipsHash: verifyCore updated.\n");
1839#endif
1840    }
1841
1842    ret = 0;
1843
1844  out:
1845
1846    if (tfm != NULL)
1847        crypto_free_shash(tfm);
1848    XFREE(desc, NULL, DYNAMIC_TYPE_TMP_BUFFER);
1849    XFREE(old_verifyCore, NULL, DYNAMIC_TYPE_TMP_BUFFER);
1850
1851    return ret;
1852}
1853
1854#endif /* HAVE_FIPS && WOLFCRYPT_FIPS_CORE_DYNAMIC_HASH_VALUE */
1855
1856#ifdef CONFIG_HAVE_KPROBES
1857
1858static WC_MAYBE_UNUSED void *my_kallsyms_lookup_name(const char *name) {
1859    static typeof(kallsyms_lookup_name) *kallsyms_lookup_name_ptr = NULL;
1860    static struct kprobe kallsyms_lookup_name_kp = {
1861        .symbol_name = "kallsyms_lookup_name"
1862    };
1863    unsigned long a;
1864
1865    if (! kallsyms_lookup_name_ptr) {
1866        int ret;
1867        kallsyms_lookup_name_kp.addr = NULL;
1868        if ((ret = register_kprobe(&kallsyms_lookup_name_kp)) != 0) {
1869#ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG
1870            pr_err_once("ERROR: register_kprobe(&kallsyms_lookup_name_kp) failed: %d\n", ret);
1871#endif
1872            return 0;
1873        }
1874        kallsyms_lookup_name_ptr = (typeof(kallsyms_lookup_name_ptr))kallsyms_lookup_name_kp.addr;
1875        unregister_kprobe(&kallsyms_lookup_name_kp);
1876        if (! kallsyms_lookup_name_ptr) {
1877#ifdef WOLFSSL_LINUXKM_VERBOSE_DEBUG
1878            pr_err_once("ERROR: kallsyms_lookup_name_kp.addr is null.\n");
1879#endif
1880            return 0;
1881        }
1882    }
1883
1884    a = kallsyms_lookup_name_ptr(name);
1885    return (void *)a;
1886}
1887
1888#endif /* CONFIG_HAVE_KPROBES */
1889
1890#ifdef HAVE_FIPS
1891
1892static ssize_t FIPS_rerun_self_test_handler(struct kobject *kobj, struct kobj_attribute *attr,
1893                                   const char *buf, size_t count)
1894{
1895    int ret;
1896
1897    (void)kobj;
1898    (void)attr;
1899
1900    /* only recognize "1" and "1\n". */
1901    if ((count < 1) || (count > 2) ||
1902        (buf[0] != '1') ||
1903        ((count == 2) && (buf[1] != '\n')))
1904    {
1905        return -EINVAL;
1906    }
1907
1908    pr_info("wolfCrypt: rerunning FIPS self-test on command.\n");
1909
1910    if (WC_SIG_IGNORE_BEGIN() >= 0) {
1911        ret = wolfCrypt_IntegrityTest_fips();
1912        (void)WC_SIG_IGNORE_END();
1913    }
1914    else {
1915        pr_err("ERROR: WC_SIG_IGNORE_BEGIN() failed.\n");
1916        ret = -1;
1917    }
1918    if (ret != 0) {
1919        pr_err("ERROR: wolfCrypt_IntegrityTest_fips: error %d\n", ret);
1920        return -EINVAL;
1921    }
1922
1923    ret = wolfCrypt_GetStatus_fips();
1924    if (ret != 0) {
1925        pr_err("ERROR: wolfCrypt_GetStatus_fips() failed with code %d: %s\n", ret, wc_GetErrorString(ret));
1926        if (ret == WC_NO_ERR_TRACE(IN_CORE_FIPS_E))
1927            return -ELIBBAD;
1928        else
1929            return -EINVAL;
1930    }
1931
1932    ret = wc_RunAllCast_fips();
1933    if (ret != 0) {
1934        pr_err("ERROR: wc_RunAllCast_fips() failed with return value %d\n", ret);
1935        return -EINVAL;
1936    }
1937
1938    pr_info("wolfCrypt FIPS re-self-test succeeded: all algorithms verified and available.\n");
1939
1940    return count;
1941}
1942
1943#ifdef FIPS_OPTEST
1944
1945typedef struct test_func_args {
1946    int return_code;
1947} test_func_args;
1948
1949static ssize_t FIPS_optest_trig_handler(struct kobject *kobj, struct kobj_attribute *attr,
1950                                   const char *buf, const size_t count)
1951{
1952    int ret;
1953    int argc;
1954    const char *argv[3];
1955    char code_buf[5];
1956    size_t corrected_count;
1957    int i;
1958
1959    (void)kobj;
1960    (void)attr;
1961
1962    /* buf may or may not have an LF at end -- tolerate both.  there is no
1963     * terminating null in either case.
1964     */
1965    if (count < 1)
1966        return -EINVAL;
1967    if (buf[count-1] == '\n')
1968        corrected_count = count - 1;
1969    else
1970        corrected_count = count;
1971    if ((corrected_count < 1) || (corrected_count > 4))
1972        return -EINVAL;
1973    memcpy(code_buf, buf, corrected_count);
1974    code_buf[corrected_count] = 0;
1975
1976    if (strspn(code_buf, "-0123456789") != corrected_count)
1977        return -EINVAL;
1978
1979    argv[0] = "./optest";
1980    argv[1] = "0";
1981    argv[2] = code_buf;
1982    argc = 3;
1983
1984    printf("OK, testing code %s\n", code_buf);
1985
1986    #ifdef WC_LINUXKM_HAVE_STACK_DEBUG
1987    {
1988        unsigned long stack_usage;
1989        stack_usage = wc_linuxkm_stack_current();
1990        pr_info("STACK INFO: usage at call to linuxkm_op_test_1(): %lu of %lu total\n", stack_usage, THREAD_SIZE);
1991        wc_linuxkm_stack_hwm_prepare(0xee);
1992    #endif
1993
1994    ret = linuxkm_op_test_1(argc, &argv[0]);
1995
1996    #ifdef WC_LINUXKM_HAVE_STACK_DEBUG
1997        stack_usage = wc_linuxkm_stack_hwm_measure_rel(0xee);
1998        pr_info("STACK INFO: rel usage by linuxkm_op_test_1(): %lu\n", stack_usage);
1999        /* shush up false stack HWM reading by kernel: */
2000        wc_linuxkm_stack_hwm_prepare(0);
2001    }
2002    #endif
2003
2004    printf("ret of op_test = %d\n", ret);
2005
2006    /* reload the library in memory and re-init state */
2007    printf("Reloading the module in memory (equivalent to power "
2008           "cycle)\n");
2009    WOLFSSL_ATOMIC_STORE(*conTestFailure_ptr, 0);
2010    for (i = 0; i < FIPS_CAST_COUNT; ++i)
2011        fipsCastStatus_put(i, FIPS_CAST_STATE_INIT);
2012    /* note, must call fipsEntry() here, not wolfCrypt_IntegrityTest_fips(),
2013     * because wc_GetCastStatus_fips(FIPS_CAST_HMAC_SHA2_256) isn't available
2014     * anymore.
2015     */
2016    if (WC_SIG_IGNORE_BEGIN() >= 0) {
2017        fipsEntry();
2018        (void)WC_SIG_IGNORE_END();
2019    }
2020    else
2021        pr_err("ERROR: WC_SIG_IGNORE_BEGIN() failed.\n");
2022    ret = wolfCrypt_GetStatus_fips();
2023    printf("Status indicator of library reload/powercycle: %d\n",
2024           ret);
2025    printf("Module status is: %d\n", wolfCrypt_GetStatus_fips());
2026    printf("Module mode is: %d\n", wolfCrypt_GetMode_fips());
2027
2028    return count;
2029}
2030
2031#endif /* FIPS_OPTEST */
2032
2033#endif /* HAVE_FIPS */