luna - wolfssl/linuxkm/linuxkm_wc

   1/* linuxkm_wc_port.h
   2 *
   3 * Copyright (C) 2006-2026 wolfSSL Inc.
   4 *
   5 * This file is part of wolfSSL.
   6 *
   7 * wolfSSL is free software; you can redistribute it and/or modify
   8 * it under the terms of the GNU General Public License as published by
   9 * the Free Software Foundation; either version 3 of the License, or
  10 * (at your option) any later version.
  11 *
  12 * wolfSSL is distributed in the hope that it will be useful,
  13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 * GNU General Public License for more details.
  16 *
  17 * You should have received a copy of the GNU General Public License
  18 * along with this program; if not, write to the Free Software
  19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
  20 */
  21
  22/* included by wolfssl/wolfcrypt/wc_port.h */
  23
  24#ifndef LINUXKM_WC_PORT_H
  25#define LINUXKM_WC_PORT_H
  26
  27    #if defined(WOLFSSL_KERNEL_VERBOSE_DEBUG) && !defined(WOLFSSL_LINUXKM_VERBOSE_DEBUG)
  28        #define WOLFSSL_LINUXKM_VERBOSE_DEBUG
  29    #endif
  30
  31    #include <linux/version.h>
  32    #include <linux/kconfig.h>
  33
  34    #if LINUX_VERSION_CODE < KERNEL_VERSION(3, 16, 0)
  35        #error Unsupported kernel.
  36    #endif
  37
  38    #if defined(HAVE_FIPS) && defined(LINUXKM_LKCAPI_REGISTER_AESXTS) && defined(CONFIG_CRYPTO_MANAGER_EXTRA_TESTS)
  39        /* CONFIG_CRYPTO_MANAGER_EXTRA_TESTS expects AES-XTS-384 to work, even when CONFIG_CRYPTO_FIPS, but FIPS 140-3 only allows AES-XTS-256 and AES-XTS-512. */
  40        #error CONFIG_CRYPTO_MANAGER_EXTRA_TESTS is incompatible with FIPS wolfCrypt AES-XTS -- please reconfigure the target kernel to disable CONFIG_CRYPTO_MANAGER_EXTRA_TESTS.
  41    #endif
  42
  43    /* The first vector set in /usr/src/linux/crypto/testmgr.h
  44     * ecdsa_nist_p192_tv_template[], ecdsa_nist_p256_tv_template[], and
  45     * ecdsa_nist_p384_tv_template[] use SHA-1 (even if CONFIG_CRYPTO_SHA1 is
  46     * disabled), and kernel module signatures frequently use SHA-1 until quite
  47     * recently (dependent on CONFIG_CRYPTO_SHA1).  If either is enabled, force
  48     * downgrade to 186-4.
  49     */
  50    #if defined(WC_FIPS_186_5_PLUS) && \
  51        (defined(CONFIG_CRYPTO_SHA1) || (defined(CONFIG_CRYPTO_MANAGER) && !defined(CONFIG_CRYPTO_MANAGER_DISABLE_TESTS))) && \
  52        (defined(LINUXKM_LKCAPI_REGISTER_ALL) || defined(LINUXKM_LKCAPI_REGISTER_ALL_KCONFIG) || defined(CONFIG_CRYPTO_ECDSA))
  53        #undef WC_FIPS_186_5_PLUS
  54        #ifdef WC_FIPS_186_5
  55            #undef WC_FIPS_186_5
  56        #else
  57            #error Unknown and incompatible FIPS 186 is enabled.
  58        #endif
  59        #define WC_FIPS_186_4
  60    #endif
  61
  62    #ifdef HAVE_CONFIG_H
  63        #ifndef PACKAGE_NAME
  64            #error wc_port.h included before config.h
  65        #endif
  66        /* config.h is autogenerated without gating, and is subject to repeat
  67         * inclusions, so gate it out here to keep autodetection masking
  68         * intact:
  69         */
  70        #undef HAVE_CONFIG_H
  71    #endif
  72
  73    /* suppress inclusion of stdint-gcc.h to avoid conflicts with Linux native
  74     * include/linux/types.h:
  75     */
  76    #define _GCC_STDINT_H
  77    #define WC_PTR_TYPE uintptr_t
  78
  79    /* needed to suppress inclusion of stdio.h in wolfssl/wolfcrypt/types.h */
  80    #define XSNPRINTF snprintf
  81
  82    /* the rigmarole around kstrtoll() here is to accommodate its
  83     * warn-unused-result attribute.
  84     *
  85     * also needed to suppress inclusion of stdlib.h in
  86     * wolfssl/wolfcrypt/types.h.
  87     */
  88    #define XATOI(s) ({                                 \
  89          long long _xatoi_res = 0;                     \
  90          int _xatoi_ret = kstrtoll(s, 10, &_xatoi_res); \
  91          if (_xatoi_ret != 0) {                        \
  92            _xatoi_res = 0;                             \
  93          }                                             \
  94          (int)_xatoi_res;                              \
  95        })
  96
  97    /* Kbuild+gcc on x86 doesn't consistently honor the default ALIGN16 on stack
  98     * objects, but gives adequate alignment with "32".
  99     */
 100    #if defined(CONFIG_X86) && !defined(ALIGN16)
 101        #define ALIGN16 __attribute__ ( (aligned (32)))
 102    #endif
 103
 104    /* kvmalloc()/kvfree() and friends added in linux commit a7c3e901, merged for 4.12.
 105     * kvrealloc() added in de2860f463, merged for 5.15, backported to 5.10.137.
 106     * moved to ultimate home (slab.h) in 8587ca6f34, merged for 5.16.
 107     *
 108     * however, until 6.12 (commit 590b9d576c), it took an extra argument,
 109     * oldsize, that makes it incompatible with traditional libc usage patterns,
 110     * so we don't try to use it.
 111     */
 112    #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 12, 0) && \
 113        !defined(DONT_HAVE_KVMALLOC) && !defined(HAVE_KVMALLOC)
 114        #define HAVE_KVMALLOC
 115    #endif
 116    #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 12, 0) && \
 117        !defined(DONT_HAVE_KVREALLOC) && !defined(HAVE_KVREALLOC)
 118        #define HAVE_KVREALLOC
 119    #endif
 120
 121    #ifdef WOLFCRYPT_ONLY
 122        #if defined(HAVE_KVMALLOC) && \
 123            !defined(DONT_USE_KVMALLOC) && !defined(USE_KVMALLOC)
 124            #define USE_KVMALLOC
 125        #endif
 126        #if defined(HAVE_KVREALLOC) && \
 127            !defined(DONT_USE_KVREALLOC) && !defined(USE_KVREALLOC)
 128            #define USE_KVREALLOC
 129        #endif
 130    #else
 131        /* functioning realloc() is needed for the TLS stack. */
 132        #if defined(HAVE_KVMALLOC) && defined(HAVE_KVREALLOC) && \
 133            !defined(DONT_USE_KVMALLOC) && !defined(DONT_USE_KVREALLOC)
 134            #ifndef USE_KVMALLOC
 135                #define USE_KVMALLOC
 136            #endif
 137            #ifndef USE_KVREALLOC
 138                #define USE_KVREALLOC
 139            #endif
 140        #endif
 141    #endif
 142
 143    #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 1, 0)
 144        /* added by 6bab69c650 */
 145        #define static_assert(expr, ...) __static_assert(expr, ##__VA_ARGS__, #expr)
 146        #define __static_assert(expr, msg, ...) _Static_assert(expr, msg)
 147    #endif
 148
 149    /* kernel printf doesn't implement fp. */
 150    #ifndef WOLFSSL_NO_FLOAT_FMT
 151        #define WOLFSSL_NO_FLOAT_FMT
 152    #endif
 153
 154    #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0)) || \
 155        (defined(RHEL_MAJOR) && \
 156         ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
 157        #define WOLFSSL_DEBUG_PRINTF_FN _printk
 158    #else
 159        #define WOLFSSL_DEBUG_PRINTF_FN printk
 160    #endif
 161
 162#ifndef WOLFSSL_LINUXKM_USE_MUTEXES
 163    struct wolfSSL_Mutex;
 164    extern int wc_lkm_LockMutex(struct wolfSSL_Mutex* m);
 165#endif
 166
 167    #ifndef WC_LINUXKM_INTR_SIGNALS
 168        #define WC_LINUXKM_INTR_SIGNALS { SIGKILL, SIGABRT, SIGHUP, SIGINT }
 169    #endif
 170    extern int wc_linuxkm_sig_ignore_begin(void);
 171    extern int wc_linuxkm_sig_ignore_end(void);
 172    extern int wc_linuxkm_check_for_intr_signals(void);
 173    #ifndef WC_LINUXKM_MAX_NS_WITHOUT_YIELD
 174        #define WC_LINUXKM_MAX_NS_WITHOUT_YIELD 1000000000
 175    #endif
 176    extern void wc_linuxkm_relax_long_loop(void);
 177
 178    #ifndef WC_SIG_IGNORE_BEGIN
 179        #define WC_SIG_IGNORE_BEGIN() wc_linuxkm_sig_ignore_begin()
 180    #endif
 181    #ifndef WC_SIG_IGNORE_END
 182        #define WC_SIG_IGNORE_END() wc_linuxkm_sig_ignore_end()
 183    #endif
 184    #ifndef WC_CHECK_FOR_INTR_SIGNALS
 185        #define WC_CHECK_FOR_INTR_SIGNALS() wc_linuxkm_check_for_intr_signals()
 186    #endif
 187    #ifndef WC_RELAX_LONG_LOOP
 188        #define WC_RELAX_LONG_LOOP() wc_linuxkm_relax_long_loop()
 189    #endif
 190
 191    enum wc_svr_flags {
 192        WC_SVR_FLAG_NONE = 0,
 193        WC_SVR_FLAG_INHIBIT = 1,
 194    };
 195
 196    #if defined(WOLFSSL_AESNI) || defined(USE_INTEL_SPEEDUP) || \
 197        defined(WOLFSSL_SP_X86_64_ASM)
 198        #ifndef CONFIG_X86
 199            #error X86 SIMD extensions requested, but CONFIG_X86 is not set.
 200        #endif
 201        #define WOLFSSL_LINUXKM_SIMD
 202        #define WOLFSSL_LINUXKM_SIMD_X86
 203        #ifndef WOLFSSL_USE_SAVE_VECTOR_REGISTERS
 204            #define WOLFSSL_USE_SAVE_VECTOR_REGISTERS
 205        #endif
 206    #elif defined(WOLFSSL_ARMASM) || defined(WOLFSSL_SP_ARM32_ASM) || \
 207          defined(WOLFSSL_SP_ARM64_ASM) || defined(WOLFSSL_SP_ARM_THUMB_ASM) ||\
 208          defined(WOLFSSL_SP_ARM_CORTEX_M_ASM)
 209        #if !defined(CONFIG_ARM) && !defined(CONFIG_ARM64)
 210            #error ARM SIMD extensions requested, but CONFIG_ARM* is not set.
 211        #endif
 212        #define WOLFSSL_LINUXKM_SIMD
 213        #define WOLFSSL_LINUXKM_SIMD_ARM
 214        #ifndef WOLFSSL_USE_SAVE_VECTOR_REGISTERS
 215            #define WOLFSSL_USE_SAVE_VECTOR_REGISTERS
 216        #endif
 217    #endif
 218
 219    #if defined(HAVE_HASHDRBG) && defined(HAVE_FIPS) && \
 220            defined(HAVE_ENTROPY_MEMUSE) && \
 221            !defined(WC_LINUXKM_WOLFENTROPY_IN_GLUE_LAYER)
 222        #define WC_LINUXKM_WOLFENTROPY_IN_GLUE_LAYER
 223    #elif defined(HAVE_HASHDRBG) && defined(HAVE_FIPS) && \
 224            (defined(HAVE_INTEL_RDSEED) || defined(HAVE_AMD_RDSEED)) && \
 225            !defined(HAVE_ENTROPY_MEMUSE) && \
 226            !defined(WC_LINUXKM_RDSEED_IN_GLUE_LAYER)
 227        #define WC_LINUXKM_RDSEED_IN_GLUE_LAYER
 228    #endif
 229    #if defined(WC_LINUXKM_WOLFENTROPY_IN_GLUE_LAYER)
 230        struct OS_Seed;
 231        extern int wc_linuxkm_GenerateSeed_wolfEntropy(struct OS_Seed* os, unsigned char* output, unsigned int sz);
 232        #define WC_GENERATE_SEED_DEFAULT wc_linuxkm_GenerateSeed_wolfEntropy
 233    #elif defined(WC_LINUXKM_RDSEED_IN_GLUE_LAYER)
 234        struct OS_Seed;
 235        extern int wc_linuxkm_GenerateSeed_IntelRD(struct OS_Seed* os, unsigned char* output, unsigned int sz);
 236        #define WC_GENERATE_SEED_DEFAULT wc_linuxkm_GenerateSeed_IntelRD
 237    #endif
 238
 239    /* setup for LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT needs to be here
 240     * to assure that calls to get_random_bytes() in random.c are gated out
 241     * (they would recurse, potentially infinitely).
 242     */
 243    #if defined(LINUXKM_LKCAPI_REGISTER_ALL) && \
 244        !defined(LINUXKM_LKCAPI_DONT_REGISTER_HASH_DRBG) && \
 245        !defined(LINUXKM_LKCAPI_DONT_REGISTER_HASH_DRBG_DEFAULT) && \
 246        !defined(NO_LINUXKM_DRBG_GET_RANDOM_BYTES) && \
 247        !defined(LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT) && \
 248        defined(HAVE_HASHDRBG)
 249        #define LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT
 250    #endif
 251
 252    #ifdef BUILDING_WOLFSSL
 253
 254    #if ((LINUX_VERSION_CODE >= KERNEL_VERSION(5, 16, 0)) || \
 255         (defined(RHEL_MAJOR) &&                                    \
 256          ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))) && \
 257        defined(CONFIG_X86)
 258        /* linux/slab.h recursively brings in linux/page-flags.h, bringing in
 259         * non-inline implementations of functions folio_flags() and
 260         * const_folio_flags().  but we can retrofit the attribute.
 261         */
 262        struct folio;
 263        static __always_inline unsigned long *folio_flags(
 264            struct folio *folio, unsigned n);
 265        #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 9, 0)
 266            static __always_inline const unsigned long *const_folio_flags(
 267                const struct folio *folio, unsigned n);
 268        #endif
 269    #endif
 270
 271    #if defined(CONFIG_MIPS) && defined(WC_SYM_RELOC_TABLES)
 272        /* __ZBOOT__ disables some unhelpful macros around the mem*() funcs in
 273         * legacy arch/mips/include/asm/string.h
 274         */
 275        #define __ZBOOT__
 276        #define memcmp __builtin_memcmp
 277        #define __ARCH_MEMCMP_NO_REDIRECT
 278        #define __ARCH_MEMCPY_NO_REDIRECT
 279        #define __builtin_memcpy memcpy
 280        extern void *memcpy(void *dest, const void *src, unsigned int n);
 281        #define __ARCH_MEMCPY_NO_REDIRECT
 282        #define __builtin_memset memset
 283        extern void *memset(void *dest, int c, unsigned int n);
 284    #endif
 285
 286    _Pragma("GCC diagnostic push");
 287
 288    /* we include all the needed kernel headers with these masked out. else
 289     * there are profuse warnings, especially on older kernels.
 290     */
 291    _Pragma("GCC diagnostic ignored \"-Wunused-parameter\"");
 292    _Pragma("GCC diagnostic ignored \"-Wpointer-arith\"");
 293    _Pragma("GCC diagnostic ignored \"-Wshadow\"");
 294    _Pragma("GCC diagnostic ignored \"-Wnested-externs\"");
 295    _Pragma("GCC diagnostic ignored \"-Wredundant-decls\"");
 296    _Pragma("GCC diagnostic ignored \"-Wsign-compare\"");
 297    _Pragma("GCC diagnostic ignored \"-Wpointer-sign\"");
 298    _Pragma("GCC diagnostic ignored \"-Wbad-function-cast\"");
 299#ifndef __clang__
 300    _Pragma("GCC diagnostic ignored \"-Wdiscarded-qualifiers\"");
 301#endif
 302    _Pragma("GCC diagnostic ignored \"-Wtype-limits\"");
 303    _Pragma("GCC diagnostic ignored \"-Wswitch-enum\"");
 304    _Pragma("GCC diagnostic ignored \"-Wcast-function-type\""); /* needed for kernel 4.14.336 */
 305    _Pragma("GCC diagnostic ignored \"-Wformat-nonliteral\""); /* needed for kernel 4.9.282 */
 306    _Pragma("GCC diagnostic ignored \"-Wattributes\"");
 307
 308    #ifdef CONFIG_KASAN
 309        #ifndef WC_SANITIZE_DISABLE
 310            #define WC_SANITIZE_DISABLE() kasan_disable_current()
 311        #endif
 312        #ifndef WC_SANITIZE_ENABLE
 313            #define WC_SANITIZE_ENABLE() kasan_enable_current()
 314        #endif
 315    #endif
 316
 317    #if defined(CONFIG_FORTIFY_SOURCE) && \
 318        !defined(WC_FORCE_LINUXKM_FORTIFY_SOURCE) && \
 319        (defined(WC_SYM_RELOC_TABLES) || \
 320         (LINUX_VERSION_CODE < KERNEL_VERSION(5, 18, 0)))
 321        /* fortify-source causes all sorts of awkward problems for the PIE
 322         * build, up to and including stubborn external references and multiple
 323         * definitions of string functions.
 324         *
 325         * fortify-source before kernel 5.18 has similar issues regardless of
 326         * PIE, around our macro-shimming of the string functions.
 327         */
 328        #undef CONFIG_FORTIFY_SOURCE
 329    #endif
 330
 331    #if defined(WC_FORCE_LINUXKM_FORTIFY_SOURCE) && \
 332        !defined(CONFIG_FORTIFY_SOURCE)
 333        #error WC_FORCE_LINUXKM_FORTIFY_SOURCE without CONFIG_FORTIFY_SOURCE.
 334    #endif
 335
 336    #if defined(WC_CONTAINERIZE_THIS) && defined(CONFIG_ARM64)
 337        /* alt_cb_patch_nops and queued_spin_lock_slowpath are defined early
 338         * to allow shimming in system headers.
 339         */
 340        /* alt_cb_patch_nops added by d926079f17, release 6.1 */
 341        #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)
 342            #define alt_cb_patch_nops my__alt_cb_patch_nops
 343        #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0) */
 344        #define queued_spin_lock_slowpath my__queued_spin_lock_slowpath
 345    #endif
 346
 347    /*
 348     * Disable ARM64 LSE atomics for out-of-tree modules.
 349     *
 350     * When CONFIG_ARM64_LSE_ATOMICS is enabled, the kernel uses static keys
 351     * (jump labels) in system_uses_lse_atomics() to choose between LSE and
 352     * LL/SC atomic implementations at runtime. These static keys generate
 353     * asm goto statements that reference .jump_table section symbols which
 354     * cannot be resolved in out-of-tree modules, causing:
 355     *   "error: impossible constraint in 'asm'"
 356     *
 357     * By undefining CONFIG_ARM64_LSE_ATOMICS here (before any kernel headers
 358     * that use atomics are included), we force use of the LL/SC fallback path
 359     * which works correctly in out-of-tree modules.
 360     */
 361    #undef CONFIG_ARM64_LSE_ATOMICS
 362
 363    #include <linux/kernel.h>
 364    #include <linux/ctype.h>
 365
 366    #if defined(CONFIG_FORTIFY_SOURCE) || defined(DEBUG_LINUXKM_FORTIFY_OVERLAY)
 367        #ifdef WC_CONTAINERIZE_THIS
 368            /* the inline definitions in fortify-string.h use non-inline
 369             * fortify_panic().
 370             */
 371            extern void __my_fortify_panic(const char *name) __noreturn __cold;
 372            #if LINUX_VERSION_CODE >= KERNEL_VERSION(6,9,0)
 373                /* see linux 3d965b33e40d9 */
 374                #define fortify_panic(func, write, avail, size, retfail) \
 375                        __my_fortify_panic(#func)
 376            #else
 377                #define fortify_panic __my_fortify_panic
 378            #endif
 379        #endif
 380
 381        /* the _FORTIFY_SOURCE macros and implementations for several string
 382         * functions are incompatible with libwolfssl, so just reimplement with
 383         * inlines and remap with macros.
 384         */
 385
 386        #define __ARCH_STRLEN_NO_REDIRECT
 387        #define __ARCH_MEMCPY_NO_REDIRECT
 388        #define __ARCH_MEMSET_NO_REDIRECT
 389        #define __ARCH_MEMMOVE_NO_REDIRECT
 390
 391        /* the inline definitions in fortify-string.h use non-inline
 392         * strlen().
 393         */
 394        static inline size_t strlen(const char *s) {
 395            const char *s_start = s;
 396            while (*s)
 397                ++s;
 398            return (size_t)((uintptr_t)s - (uintptr_t)s_start);
 399        }
 400
 401        #include <linux/string.h>
 402
 403        #undef strlen
 404        #define strlen(s) \
 405            ((__builtin_constant_p(s) && __builtin_constant_p(*(s))) ? \
 406             (sizeof(s) - 1) : strlen(s))
 407
 408        static inline void *my_memcpy(void *dest, const void *src, size_t n) {
 409            if (! (((uintptr_t)dest | (uintptr_t)src | (uintptr_t)n)
 410                   & (uintptr_t)(sizeof(uintptr_t) - 1)))
 411            {
 412                uintptr_t *src_longs = (uintptr_t *)src,
 413                    *dest_longs = (uintptr_t *)dest,
 414                    *endp = (uintptr_t *)((u8 *)src + n);
 415                while (src_longs < endp)
 416                    *dest_longs++ = *src_longs++;
 417            } else {
 418                u8 *src_bytes = (u8 *)src,
 419                    *dest_bytes = (u8 *)dest,
 420                    *endp = src_bytes + n;
 421                while (src_bytes < endp)
 422                    *dest_bytes++ = *src_bytes++;
 423            }
 424            return dest;
 425        }
 426        #undef memcpy
 427        #define memcpy my_memcpy
 428
 429        static inline void *my_memset(void *dest, int c, size_t n) {
 430            if (! (((uintptr_t)dest | (uintptr_t)n)
 431                   & (uintptr_t)(sizeof(uintptr_t) - 1)))
 432            {
 433                uintptr_t c_long = __builtin_choose_expr(
 434                    sizeof(uintptr_t) == 8,
 435                    (uintptr_t)(u8)c * 0x0101010101010101UL,
 436                    (uintptr_t)(u8)c * 0x01010101U
 437                    );
 438                uintptr_t *dest_longs = (uintptr_t *)dest,
 439                    *endp = (uintptr_t *)((u8 *)dest_longs + n);
 440                while (dest_longs < endp)
 441                    *dest_longs++ = c_long;
 442            } else {
 443                u8 *dest_bytes = (u8 *)dest, *endp = dest_bytes + n;
 444                while (dest_bytes < endp)
 445                    *dest_bytes++ = (u8)c;
 446            }
 447            return dest;
 448        }
 449        #undef memset
 450        #define memset my_memset
 451
 452        static inline void *my_memmove(void *dest, const void *src, size_t n) {
 453            if (n == 0)
 454                return dest;
 455            if (! (((uintptr_t)dest | (uintptr_t)src | (uintptr_t)n)
 456                   & (uintptr_t)(sizeof(uintptr_t) - 1)))
 457            {
 458                uintptr_t *src_longs = (uintptr_t *)src,
 459                    *dest_longs = (uintptr_t *)dest;
 460                n >>= __builtin_choose_expr(
 461                    sizeof(uintptr_t) == 8,
 462                    3U,
 463                    2U);
 464                if (src_longs < dest_longs) {
 465                    uintptr_t *startp = src_longs;
 466                    src_longs += n - 1;
 467                    dest_longs += n - 1;
 468                    while (src_longs >= startp)
 469                        *dest_longs-- = *src_longs--;
 470                } else if (src_longs > dest_longs) {
 471                    uintptr_t *endp = src_longs + n;
 472                    while (src_longs < endp)
 473                        *dest_longs++ = *src_longs++;
 474                }
 475            } else {
 476                u8 *src_bytes = (u8 *)src, *dest_bytes = (u8 *)dest;
 477                if (src_bytes < dest_bytes) {
 478                    u8 *startp = src_bytes;
 479                    src_bytes += n - 1;
 480                    dest_bytes += n - 1;
 481                    while (src_bytes >= startp)
 482                        *dest_bytes-- = *src_bytes--;
 483                } else if (src_bytes > dest_bytes) {
 484                    u8 *endp = src_bytes + n;
 485                    while (src_bytes < endp)
 486                        *dest_bytes++ = *src_bytes++;
 487                }
 488            }
 489            return dest;
 490        }
 491        #undef memmove
 492        #define memmove my_memmove
 493
 494    #else /* !CONFIG_FORTIFY_SOURCE */
 495
 496        #include <linux/string.h>
 497
 498    #endif /* !CONFIG_FORTIFY_SOURCE */
 499
 500    #ifndef WC_CONTAINERIZE_THIS
 501        #include <linux/init.h>
 502        #include <linux/module.h>
 503        #include <linux/moduleparam.h>
 504        #include <linux/delay.h>
 505    #endif
 506
 507    #if defined(HAVE_KVMALLOC) && \
 508        (LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0)) && \
 509        !(defined(RHEL_MAJOR) && ((RHEL_MAJOR > 9) || \
 510                                  ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
 511        /* before 5.16, the kvmalloc_node() and kvfree() prototypes were in
 512         * mm.h.  however, mm.h brings in static, but not inline, pmd_to_page(),
 513         * with direct references to global vmem variables.
 514         */
 515        #ifdef WC_CONTAINERIZE_THIS
 516            #include <linux/mm_types.h>
 517            #if USE_SPLIT_PMD_PTLOCKS
 518            static __always_inline struct page *pmd_to_page(pmd_t *pmd);
 519            #endif
 520        #endif
 521        #include <linux/mm.h>
 522    #endif
 523
 524#ifndef WC_CONTAINERIZE_THIS
 525    #include <linux/kthread.h>
 526    #include <linux/net.h>
 527    #ifndef WOLFCRYPT_ONLY
 528        #include <linux/inet.h>
 529        static inline int wc_linuxkm_inet_pton(int af, const char *src, void *dst)
 530        {
 531            int ret;
 532
 533            if (!src || !dst)
 534                return -EFAULT;
 535
 536            switch (af) {
 537            case AF_INET:
 538                ret = in4_pton(src, -1, (u8 *)dst, '\0', NULL);
 539                return ret == 1 ? 1 : 0;
 540
 541            case AF_INET6:
 542                ret = in6_pton(src, -1, (u8 *)dst, '\0', NULL);
 543                return ret == 1 ? 1 : 0;
 544
 545            default:
 546                return -EAFNOSUPPORT;
 547            }
 548        }
 549        #define XINET_PTON(af, src, dst) wc_linuxkm_inet_pton(af, src, dst)
 550    #endif /* !WOLFCRYPT_ONLY */
 551#endif /* !WC_CONTAINERIZE_THIS */
 552
 553    #if defined(WC_SYM_RELOC_TABLES) && defined(DEBUG_LINUXKM_PIE_SUPPORT) && \
 554        !defined(WC_LINUXKM_SUPPORT_DUMP_TO_FILE)
 555        #define WC_LINUXKM_SUPPORT_DUMP_TO_FILE
 556    #endif
 557
 558    #ifdef WC_LINUXKM_SUPPORT_DUMP_TO_FILE
 559        #include <linux/fs.h>
 560        #include <linux/uaccess.h>
 561    #endif
 562    #include <linux/slab.h>
 563    #include <linux/sched.h>
 564    #if __has_include(<linux/sched/task_stack.h>)
 565        /* for task_stack_page() */
 566        #include <linux/sched/task_stack.h>
 567    #endif
 568    #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 14, 0)
 569        /* for signal_pending() */
 570        #include <linux/sched/signal.h>
 571        /* for local_clock() */
 572        #include <linux/sched/clock.h>
 573    #endif
 574    #include <linux/random.h>
 575
 576    #if !defined(WC_CONTAINERIZE_THIS) && defined(CONFIG_HAVE_KPROBES)
 577        #include <linux/kprobes.h>
 578    #endif
 579
 580    #ifdef LINUXKM_LKCAPI_REGISTER
 581        /* the LKCAPI assumes that expanded encrypt and decrypt keys will stay
 582         * loaded simultaneously, and the Linux in-tree implementations have two
 583         * AES key structs in each context, one for each direction.  in
 584         * linuxkm/lkcapi_aes_glue.c, we do the same
 585         * thing with "struct km_AesCtx".  however, wolfCrypt struct AesXts
 586         * already has two AES expanded keys, the main and tweak, and the tweak
 587         * is always used in the encrypt direction regardless of the main
 588         * direction.  to avoid allocating and computing a duplicate second
 589         * tweak encrypt key, we set
 590         * WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS, which adds a second
 591         * Aes slot to wolfCrypt's struct AesXts, and activates support for
 592         * AES_ENCRYPTION_AND_DECRYPTION on AES-XTS.
 593         */
 594        #ifndef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
 595            #define WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS
 596        #endif
 597
 598        #ifndef WC_CONTAINERIZE_THIS
 599            #include <linux/crypto.h>
 600            #include <linux/scatterlist.h>
 601            #include <crypto/scatterwalk.h>
 602            #include <crypto/internal/aead.h>
 603            #include <crypto/internal/hash.h>
 604            #include <crypto/internal/rng.h>
 605            #include <crypto/internal/skcipher.h>
 606            #include <crypto/internal/akcipher.h>
 607            #include <crypto/internal/kpp.h>
 608            #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 13, 0)
 609                #include <crypto/internal/sig.h>
 610            #endif /* linux ver >= 6.13 */
 611
 612            #if defined(_LINUX_REFCOUNT_H) || defined(_LINUX_REFCOUNT_TYPES_H)
 613                static inline int wc_lkm_refcount_to_int(refcount_t *refcount) {
 614                    _Pragma("GCC diagnostic push");
 615                    _Pragma("GCC diagnostic ignored \"-Wnested-externs\"");
 616                    return atomic_read(&(refcount->refs));
 617                    _Pragma("GCC diagnostic pop");
 618                }
 619            #else
 620                static inline int wc_lkm_refcount_to_int(atomic_t *refcount) {
 621                    _Pragma("GCC diagnostic push");
 622                    _Pragma("GCC diagnostic ignored \"-Wnested-externs\"");
 623                    return atomic_read(&refcount);
 624                    _Pragma("GCC diagnostic pop");
 625                }
 626            #endif
 627            #define WC_LKM_REFCOUNT_TO_INT(refcount) wc_lkm_refcount_to_int(&(refcount))
 628        #endif /* !WC_CONTAINERIZE_THIS */
 629
 630    #endif /* LINUXKM_LKCAPI_REGISTER */
 631
 632    /* benchmarks.c uses floating point math, so needs a working
 633     * SAVE_VECTOR_REGISTERS().
 634     */
 635    #if defined(WOLFSSL_LINUXKM_BENCHMARKS) && \
 636        !defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)
 637        #define WOLFSSL_USE_SAVE_VECTOR_REGISTERS
 638    #endif
 639
 640    #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && \
 641        defined(CONFIG_X86)
 642
 643        extern __must_check int allocate_wolfcrypt_linuxkm_fpu_states(void);
 644        extern void free_wolfcrypt_linuxkm_fpu_states(void);
 645        WOLFSSL_API __must_check int wc_can_save_vector_registers_x86(void);
 646        WOLFSSL_API __must_check int wc_save_vector_registers_x86(enum wc_svr_flags flags);
 647        WOLFSSL_API void wc_restore_vector_registers_x86(enum wc_svr_flags flags);
 648
 649        #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
 650            #include <asm/i387.h>
 651            #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 13, 0)
 652                /* added by a62b01cd6c */
 653                #include <asm-generic/simd.h>
 654            #endif
 655        #else
 656            #include <asm/simd.h>
 657            #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 10, 0)
 658                /* added by 266d051601 */
 659                #include <crypto/internal/simd.h>
 660            #endif
 661        #endif
 662        #ifndef CAN_SAVE_VECTOR_REGISTERS
 663            #ifdef DEBUG_VECTOR_REGISTER_ACCESS_FUZZING
 664                #define CAN_SAVE_VECTOR_REGISTERS() (wc_can_save_vector_registers_x86() && (SAVE_VECTOR_REGISTERS2_fuzzer() == 0))
 665            #else
 666                #define CAN_SAVE_VECTOR_REGISTERS() wc_can_save_vector_registers_x86()
 667            #endif
 668        #endif
 669        #ifndef SAVE_VECTOR_REGISTERS
 670            #define SAVE_VECTOR_REGISTERS(fail_clause) {     \
 671                int _svr_ret = wc_save_vector_registers_x86(WC_SVR_FLAG_NONE); \
 672                if (_svr_ret != 0) {                         \
 673                    fail_clause                              \
 674                }                                            \
 675            }
 676        #endif
 677        #ifndef SAVE_VECTOR_REGISTERS2
 678            #ifdef DEBUG_VECTOR_REGISTER_ACCESS_FUZZING
 679                #define SAVE_VECTOR_REGISTERS2() ({                    \
 680                    int _fuzzer_ret = SAVE_VECTOR_REGISTERS2_fuzzer(); \
 681                    (_fuzzer_ret == 0) ?                               \
 682                     wc_save_vector_registers_x86(WC_SVR_FLAG_NONE) :  \
 683                     _fuzzer_ret;                                      \
 684                })
 685            #else
 686                #define SAVE_VECTOR_REGISTERS2() wc_save_vector_registers_x86(WC_SVR_FLAG_NONE)
 687            #endif
 688        #endif
 689        #ifndef RESTORE_VECTOR_REGISTERS
 690            #define RESTORE_VECTOR_REGISTERS() wc_restore_vector_registers_x86(WC_SVR_FLAG_NONE)
 691        #endif
 692
 693        #ifndef DISABLE_VECTOR_REGISTERS
 694            #define DISABLE_VECTOR_REGISTERS() wc_save_vector_registers_x86(WC_SVR_FLAG_INHIBIT)
 695        #endif
 696        #ifndef REENABLE_VECTOR_REGISTERS
 697            #define REENABLE_VECTOR_REGISTERS() wc_restore_vector_registers_x86(WC_SVR_FLAG_INHIBIT)
 698        #endif
 699
 700    #elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && (defined(CONFIG_ARM) || defined(CONFIG_ARM64))
 701
 702        #error kernel module ARM SIMD is not yet tested or usable.
 703
 704        #include <asm/fpsimd.h>
 705
 706        static WARN_UNUSED_RESULT inline int save_vector_registers_arm(void)
 707        {
 708            preempt_disable();
 709            if (! may_use_simd()) {
 710                preempt_enable();
 711                return BAD_STATE_E;
 712            } else {
 713                fpsimd_preserve_current_state();
 714                return 0;
 715            }
 716        }
 717        static inline void restore_vector_registers_arm(void)
 718        {
 719            fpsimd_restore_current_state();
 720            preempt_enable();
 721        }
 722
 723        #ifndef SAVE_VECTOR_REGISTERS
 724            #define SAVE_VECTOR_REGISTERS(fail_clause) { int _svr_ret = save_vector_registers_arm(); if (_svr_ret != 0) { fail_clause } }
 725        #endif
 726        #ifndef SAVE_VECTOR_REGISTERS2
 727            #define SAVE_VECTOR_REGISTERS2() save_vector_registers_arm()
 728        #endif
 729        #ifndef CAN_SAVE_VECTOR_REGISTERS
 730            #define CAN_SAVE_VECTOR_REGISTERS() can_save_vector_registers_arm()
 731        #endif
 732        #ifndef RESTORE_VECTOR_REGISTERS
 733            #define RESTORE_VECTOR_REGISTERS() restore_vector_registers_arm()
 734        #endif
 735
 736    #elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)
 737        #error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture.
 738    #endif /* WOLFSSL_USE_SAVE_VECTOR_REGISTERS */
 739
 740    _Pragma("GCC diagnostic pop");
 741
 742    /* avoid -Wpointer-arith, encountered when -DCONFIG_FORTIFY_SOURCE */
 743    #undef __is_constexpr
 744    #define __is_constexpr(x) __builtin_constant_p(x)
 745
 746    /* the kernel uses -std=c89, but not -pedantic, and makes full use of anon
 747     * structs/unions, so we should too.
 748     */
 749    #define HAVE_ANONYMOUS_INLINE_AGGREGATES 1
 750
 751    #define NO_THREAD_LS
 752    #define NO_ATTRIBUTE_CONSTRUCTOR
 753
 754    #ifdef HAVE_FIPS
 755        extern int wolfCrypt_FIPS_first(void);
 756        extern int wolfCrypt_FIPS_last(void);
 757        extern const unsigned int wolfCrypt_FIPS_ro_start[];
 758        extern const unsigned int wolfCrypt_FIPS_ro_end[];
 759        #if FIPS_VERSION3_GE(6,0,0)
 760#ifndef NO_AES
 761            extern int wolfCrypt_FIPS_AES_sanity(void);
 762            extern const unsigned int wolfCrypt_FIPS_aes_ro_sanity[2];
 763#if defined(WOLFSSL_CMAC) && defined(WOLFSSL_AES_DIRECT)
 764            extern int wolfCrypt_FIPS_CMAC_sanity(void);
 765            extern const unsigned int wolfCrypt_FIPS_cmac_ro_sanity[2];
 766#endif
 767#endif
 768#ifndef NO_DH
 769            extern int wolfCrypt_FIPS_DH_sanity(void);
 770            extern const unsigned int wolfCrypt_FIPS_dh_ro_sanity[2];
 771#endif
 772#ifdef HAVE_ECC
 773            extern int wolfCrypt_FIPS_ECC_sanity(void);
 774            extern const unsigned int wolfCrypt_FIPS_ecc_ro_sanity[2];
 775#endif
 776#ifdef HAVE_ED25519
 777            extern int wolfCrypt_FIPS_ED25519_sanity(void);
 778            extern const unsigned int wolfCrypt_FIPS_ed25519_ro_sanity[2];
 779#endif
 780#ifdef HAVE_ED448
 781            extern int wolfCrypt_FIPS_ED448_sanity(void);
 782            extern const unsigned int wolfCrypt_FIPS_ed448_ro_sanity[2];
 783#endif
 784            extern int wolfCrypt_FIPS_HMAC_sanity(void);
 785            extern const unsigned int wolfCrypt_FIPS_hmac_ro_sanity[2];
 786#ifndef NO_KDF
 787            extern int wolfCrypt_FIPS_KDF_sanity(void);
 788            extern const unsigned int wolfCrypt_FIPS_kdf_ro_sanity[2];
 789#endif
 790#ifdef HAVE_PBKDF2
 791            extern int wolfCrypt_FIPS_PBKDF_sanity(void);
 792            extern const unsigned int wolfCrypt_FIPS_pbkdf_ro_sanity[2];
 793#endif
 794#ifdef HAVE_HASHDRBG
 795            extern int wolfCrypt_FIPS_DRBG_sanity(void);
 796            extern const unsigned int wolfCrypt_FIPS_drbg_ro_sanity[2];
 797#endif
 798#ifndef NO_RSA
 799            extern int wolfCrypt_FIPS_RSA_sanity(void);
 800            extern const unsigned int wolfCrypt_FIPS_rsa_ro_sanity[2];
 801#endif
 802#ifndef NO_SHA
 803            extern int wolfCrypt_FIPS_SHA_sanity(void);
 804            extern const unsigned int wolfCrypt_FIPS_sha_ro_sanity[2];
 805#endif
 806#ifndef NO_SHA256
 807            extern int wolfCrypt_FIPS_SHA256_sanity(void);
 808            extern const unsigned int wolfCrypt_FIPS_sha256_ro_sanity[2];
 809#endif
 810#ifdef WOLFSSL_SHA512
 811            extern int wolfCrypt_FIPS_SHA512_sanity(void);
 812            extern const unsigned int wolfCrypt_FIPS_sha512_ro_sanity[2];
 813#endif
 814#ifdef WOLFSSL_SHA3
 815            extern int wolfCrypt_FIPS_SHA3_sanity(void);
 816            extern const unsigned int wolfCrypt_FIPS_sha3_ro_sanity[2];
 817#endif
 818            extern int wolfCrypt_FIPS_FT_sanity(void);
 819            extern const unsigned int wolfCrypt_FIPS_ft_ro_sanity[2];
 820            extern const unsigned int wolfCrypt_FIPS_f_ro_sanity[2];
 821            extern int wc_RunAllCast_fips(void);
 822        #endif
 823    #endif
 824
 825    #if !defined(WOLFCRYPT_ONLY) && !defined(NO_CERTS)
 826        /* work around backward dependency of asn.c on ssl.c. */
 827        struct Signer;
 828        struct Signer *GetCA(void *signers, unsigned char *hash);
 829        #ifndef NO_SKID
 830            struct Signer *GetCAByName(void* signers, unsigned char *hash);
 831            #ifdef HAVE_OCSP
 832                struct Signer* GetCAByKeyHash(void* vp, const unsigned char* keyHash);
 833            #endif /* HAVE_OCSP */
 834            #ifdef WOLFSSL_AKID_NAME
 835                #ifdef WOLFSSL_API_PREFIX_MAP
 836                    #define GetCAByAKID wolfSSL_GetCAByAKID
 837                #endif
 838                struct Signer* GetCAByAKID(void* vp, const unsigned char* issuer,
 839                                           unsigned int issuerSz,
 840                                           const unsigned char* serial,
 841                                           unsigned int serialSz);
 842            #endif
 843        #endif /* NO_SKID */
 844
 845        #if defined(OPENSSL_EXTRA) || defined(OPENSSL_EXTRA_X509_SMALL)
 846            struct WOLFSSL_X509_NAME;
 847            extern int wolfSSL_X509_NAME_add_entry_by_NID(struct WOLFSSL_X509_NAME *name, int nid,
 848                                               int type, const unsigned char *bytes,
 849                                               int len, int loc, int set);
 850            extern void wolfSSL_X509_NAME_free(struct WOLFSSL_X509_NAME* name);
 851            extern struct WOLFSSL_X509_NAME* wolfSSL_X509_NAME_new_ex(void *heap);
 852        #endif /* OPENSSL_EXTRA || OPENSSL_EXTRA_X509_SMALL */
 853
 854    #endif /* !WOLFCRYPT_ONLY && !NO_CERTS */
 855
 856    #if defined(WC_CONTAINERIZE_THIS) && !defined(WC_SYM_RELOC_TABLES)
 857        #error "compiling -DWC_CONTAINERIZE_THIS requires relocation tables."
 858    #endif
 859
 860    #ifdef WC_SYM_RELOC_TABLES
 861
 862    extern __attribute__((error("uncallable fencepost"))) int __wc_text_start(void);
 863    extern __attribute__((error("uncallable fencepost"))) int __wc_text_end(void);
 864    #ifdef HAVE_FIPS
 865        static_assert(__builtin_types_compatible_p(typeof(__wc_text_start), typeof(wolfCrypt_FIPS_first)));
 866        static_assert(__builtin_types_compatible_p(typeof(__wc_text_end), typeof(wolfCrypt_FIPS_last)));
 867    #endif
 868    extern const unsigned int __wc_rodata_start[], __wc_rodata_end[];
 869    #ifdef HAVE_FIPS
 870        static_assert(__builtin_types_compatible_p(typeof(__wc_rodata_start), typeof(wolfCrypt_FIPS_ro_start)));
 871        static_assert(__builtin_types_compatible_p(typeof(__wc_rodata_end), typeof(wolfCrypt_FIPS_ro_end)));
 872    #endif
 873
 874    extern const u8
 875        __wc_rwdata_start[],
 876        __wc_rwdata_end[],
 877        __wc_bss_start[],
 878        __wc_bss_end[];
 879
 880    extern ssize_t wc_linuxkm_normalize_relocations(
 881        const u8 *seg_in,
 882        size_t *seg_in_out_len,
 883        u8 *seg_out,
 884        ssize_t *cur_index_p);
 885
 886    extern ssize_t wc_linuxkm_normalize_relocations_noresize(
 887        const u8 *seg_in,
 888        size_t seg_in_len,
 889        u8 *seg_out,
 890        ssize_t *cur_index_p);
 891
 892    #ifndef WOLFSSL_SEGMENT_CANONICALIZER_BUFSIZ
 893        #define WOLFSSL_SEGMENT_CANONICALIZER_BUFSIZ 8192
 894    #endif
 895
 896    #ifndef WOLFSSL_SEGMENT_CANONICALIZER
 897        #define WOLFSSL_SEGMENT_CANONICALIZER(seg_in, seg_in_out_len, seg_out, cur_index_p) \
 898            wc_linuxkm_normalize_relocations(seg_in, seg_in_out_len, seg_out, cur_index_p)
 899    #endif
 900
 901    /* backward-compatible wrappers */
 902    #ifndef WOLFSSL_TEXT_SEGMENT_CANONICALIZER
 903        #define WOLFSSL_TEXT_SEGMENT_CANONICALIZER(text_in, text_in_len, text_out, cur_index_p) \
 904            wc_linuxkm_normalize_relocations_noresize(text_in, text_in_len, text_out, cur_index_p)
 905    #endif
 906    #ifndef WOLFSSL_TEXT_SEGMENT_CANONICALIZER_BUFSIZ
 907        #define WOLFSSL_TEXT_SEGMENT_CANONICALIZER_BUFSIZ WOLFSSL_SEGMENT_CANONICALIZER_BUFSIZ
 908    #endif
 909
 910#ifdef CONFIG_MIPS
 911    #undef __ARCH_MEMCMP_NO_REDIRECT
 912    #undef memcmp
 913    extern int memcmp(const void *s1, const void *s2, size_t n);
 914#endif
 915
 916    struct wolfssl_linuxkm_pie_redirect_table {
 917    #ifdef HAVE_FIPS
 918        typeof(wc_linuxkm_normalize_relocations) *wc_linuxkm_normalize_relocations;
 919        typeof(wc_linuxkm_normalize_relocations_noresize) *wc_linuxkm_normalize_relocations_noresize;
 920    #endif
 921
 922    #ifndef __ARCH_MEMCMP_NO_REDIRECT
 923        typeof(memcmp) *memcmp;
 924    #endif
 925    #ifndef __ARCH_MEMCPY_NO_REDIRECT
 926        typeof(memcpy) *memcpy;
 927    #endif
 928    #ifndef __ARCH_MEMSET_NO_REDIRECT
 929        typeof(memset) *memset;
 930    #endif
 931    #ifndef __ARCH_MEMMOVE_NO_REDIRECT
 932        typeof(memmove) *memmove;
 933    #endif
 934    #ifndef __ARCH_STRCMP_NO_REDIRECT
 935        typeof(strcmp) *strcmp;
 936    #endif
 937    #ifndef __ARCH_STRNCMP_NO_REDIRECT
 938        typeof(strncmp) *strncmp;
 939    #endif
 940    #ifndef __ARCH_STRCASECMP_NO_REDIRECT
 941        typeof(strcasecmp) *strcasecmp;
 942    #endif
 943    #ifndef __ARCH_STRNCASECMP_NO_REDIRECT
 944        typeof(strncasecmp) *strncasecmp;
 945    #endif
 946    #ifndef __ARCH_STRLEN_NO_REDIRECT
 947        typeof(strlen) *strlen;
 948    #endif
 949    #ifndef __ARCH_STRSTR_NO_REDIRECT
 950        typeof(strstr) *strstr;
 951    #endif
 952    #ifndef __ARCH_STRNCPY_NO_REDIRECT
 953        typeof(strncpy) *strncpy;
 954    #endif
 955    #ifndef __ARCH_STRNCAT_NO_REDIRECT
 956        typeof(strncat) *strncat;
 957    #endif
 958        typeof(kstrtoll) *kstrtoll;
 959
 960        #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0)) || \
 961            (defined(RHEL_MAJOR) && \
 962             ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
 963            typeof(_printk) *_printk;
 964        #else
 965            typeof(printk) *printk;
 966        #endif
 967
 968#ifdef CONFIG_FORTIFY_SOURCE
 969        typeof(__warn_printk) *__warn_printk;
 970#endif
 971
 972        typeof(snprintf) *snprintf;
 973
 974        const unsigned char *_ctype;
 975
 976#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0)
 977        typeof(kmalloc_noprof) *kmalloc_noprof;
 978        typeof(krealloc_node_align_noprof) *krealloc_node_align_noprof;
 979        typeof(kzalloc_noprof) *kzalloc_noprof;
 980        typeof(__kvmalloc_node_noprof) *__kvmalloc_node_noprof;
 981        typeof(__kmalloc_cache_noprof) *__kmalloc_cache_noprof;
 982        #ifdef HAVE_KVREALLOC
 983            typeof(kvrealloc_node_align_noprof) *kvrealloc_node_align_noprof;
 984        #endif
 985#elif LINUX_VERSION_CODE >= KERNEL_VERSION(6, 11, 0)
 986        typeof(kmalloc_noprof) *kmalloc_noprof;
 987        typeof(krealloc_noprof) *krealloc_noprof;
 988        typeof(kzalloc_noprof) *kzalloc_noprof;
 989        typeof(__kvmalloc_node_noprof) *__kvmalloc_node_noprof;
 990        typeof(__kmalloc_cache_noprof) *__kmalloc_cache_noprof;
 991        #ifdef HAVE_KVREALLOC
 992            typeof(kvrealloc_noprof) *kvrealloc_noprof;
 993        #endif
 994#elif LINUX_VERSION_CODE >= KERNEL_VERSION(6, 10, 0)
 995        typeof(kmalloc_noprof) *kmalloc_noprof;
 996        typeof(krealloc_noprof) *krealloc_noprof;
 997        typeof(kzalloc_noprof) *kzalloc_noprof;
 998        typeof(kvmalloc_node_noprof) *kvmalloc_node_noprof;
 999        typeof(kmalloc_trace_noprof) *kmalloc_trace_noprof;
1000        #ifdef HAVE_KVREALLOC
1001            typeof(kvrealloc_noprof) *kvrealloc_noprof;
1002        #endif
1003#else /* <6.10.0 */
1004        typeof(kmalloc) *kmalloc;
1005        typeof(krealloc) *krealloc;
1006        #ifdef HAVE_KVMALLOC
1007            typeof(kvmalloc_node) *kvmalloc_node;
1008        #endif
1009        #ifdef HAVE_KVREALLOC
1010            typeof(kvrealloc) *kvrealloc;
1011        #endif
1012        #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0) || \
1013            (defined(RHEL_MAJOR) &&                                    \
1014             ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
1015            typeof(kmalloc_trace) *kmalloc_trace;
1016        #else
1017            typeof(kmem_cache_alloc_trace) *kmem_cache_alloc_trace;
1018            typeof(kmalloc_order_trace) *kmalloc_order_trace;
1019        #endif
1020#endif /* <6.10.0 */
1021        #ifdef HAVE_KVMALLOC
1022        typeof(kvfree) *kvfree;
1023        #endif
1024        typeof(kfree) *kfree;
1025        typeof(ksize) *ksize;
1026
1027#ifndef LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT
1028        typeof(get_random_bytes) *get_random_bytes;
1029#endif
1030        #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
1031            typeof(getnstimeofday) *getnstimeofday;
1032        #elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
1033            typeof(current_kernel_time64) *current_kernel_time64;
1034        #else
1035            typeof(ktime_get_coarse_real_ts64) *ktime_get_coarse_real_ts64;
1036        #endif
1037
1038        struct task_struct *(*get_current)(void);
1039
1040        #ifdef WOLFSSL_USE_SAVE_VECTOR_REGISTERS
1041
1042            #ifdef CONFIG_X86
1043                typeof(allocate_wolfcrypt_linuxkm_fpu_states) *allocate_wolfcrypt_linuxkm_fpu_states;
1044                typeof(wc_can_save_vector_registers_x86) *wc_can_save_vector_registers_x86;
1045                typeof(free_wolfcrypt_linuxkm_fpu_states) *free_wolfcrypt_linuxkm_fpu_states;
1046                typeof(wc_restore_vector_registers_x86) *wc_restore_vector_registers_x86;
1047                typeof(wc_save_vector_registers_x86) *wc_save_vector_registers_x86;
1048            #else /* !CONFIG_X86 */
1049                #error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture.
1050            #endif /* arch */
1051
1052        #endif /* WOLFSSL_USE_SAVE_VECTOR_REGISTERS */
1053
1054        #ifndef CONFIG_PREEMPT_RT
1055            typeof(__mutex_init) *__mutex_init;
1056        #else
1057            typeof(__rt_mutex_init) *__rt_mutex_init;
1058            typeof(rt_mutex_base_init) *rt_mutex_base_init;
1059            typeof(rt_spin_lock) *rt_spin_lock;
1060            typeof(rt_spin_unlock) *rt_spin_unlock;
1061        #endif
1062
1063        #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
1064            typeof(mutex_lock_nested) *mutex_lock_nested;
1065        #else
1066            typeof(mutex_lock) *mutex_lock;
1067        #endif
1068        typeof(mutex_unlock) *mutex_unlock;
1069        #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
1070            typeof(mutex_destroy) *mutex_destroy;
1071        #endif
1072
1073        #ifdef HAVE_FIPS
1074        typeof(wolfCrypt_FIPS_first) *wolfCrypt_FIPS_first;
1075        typeof(wolfCrypt_FIPS_last) *wolfCrypt_FIPS_last;
1076        typeof(wolfCrypt_FIPS_ro_start) *wolfCrypt_FIPS_ro_start;
1077        typeof(wolfCrypt_FIPS_ro_end) *wolfCrypt_FIPS_ro_end;
1078        #if FIPS_VERSION3_GE(6,0,0)
1079#ifndef NO_AES
1080            typeof(wolfCrypt_FIPS_AES_sanity) *wolfCrypt_FIPS_AES_sanity;
1081            typeof(wolfCrypt_FIPS_aes_ro_sanity) *wolfCrypt_FIPS_aes_ro_sanity;
1082#if defined(WOLFSSL_CMAC) && defined(WOLFSSL_AES_DIRECT)
1083            typeof(wolfCrypt_FIPS_CMAC_sanity) *wolfCrypt_FIPS_CMAC_sanity;
1084            typeof(wolfCrypt_FIPS_cmac_ro_sanity) *wolfCrypt_FIPS_cmac_ro_sanity;
1085#endif
1086#endif
1087#ifndef NO_DH
1088            typeof(wolfCrypt_FIPS_DH_sanity) *wolfCrypt_FIPS_DH_sanity;
1089            typeof(wolfCrypt_FIPS_dh_ro_sanity) *wolfCrypt_FIPS_dh_ro_sanity;
1090#endif
1091#ifdef HAVE_ECC
1092            typeof(wolfCrypt_FIPS_ECC_sanity) *wolfCrypt_FIPS_ECC_sanity;
1093            typeof(wolfCrypt_FIPS_ecc_ro_sanity) *wolfCrypt_FIPS_ecc_ro_sanity;
1094#endif
1095#ifdef HAVE_ED25519
1096            typeof(wolfCrypt_FIPS_ED25519_sanity) *wolfCrypt_FIPS_ED25519_sanity;
1097            typeof(wolfCrypt_FIPS_ed25519_ro_sanity) *wolfCrypt_FIPS_ed25519_ro_sanity;
1098#endif
1099#ifdef HAVE_ED448
1100            typeof(wolfCrypt_FIPS_ED448_sanity) *wolfCrypt_FIPS_ED448_sanity;
1101            typeof(wolfCrypt_FIPS_ed448_ro_sanity) *wolfCrypt_FIPS_ed448_ro_sanity;
1102#endif
1103            typeof(wolfCrypt_FIPS_HMAC_sanity) *wolfCrypt_FIPS_HMAC_sanity;
1104            typeof(wolfCrypt_FIPS_hmac_ro_sanity) *wolfCrypt_FIPS_hmac_ro_sanity;
1105#ifndef NO_KDF
1106            typeof(wolfCrypt_FIPS_KDF_sanity) *wolfCrypt_FIPS_KDF_sanity;
1107            typeof(wolfCrypt_FIPS_kdf_ro_sanity) *wolfCrypt_FIPS_kdf_ro_sanity;
1108#endif
1109#ifdef HAVE_PBKDF2
1110            typeof(wolfCrypt_FIPS_PBKDF_sanity) *wolfCrypt_FIPS_PBKDF_sanity;
1111            typeof(wolfCrypt_FIPS_pbkdf_ro_sanity) *wolfCrypt_FIPS_pbkdf_ro_sanity;
1112#endif
1113#ifdef HAVE_HASHDRBG
1114            typeof(wolfCrypt_FIPS_DRBG_sanity) *wolfCrypt_FIPS_DRBG_sanity;
1115            typeof(wolfCrypt_FIPS_drbg_ro_sanity) *wolfCrypt_FIPS_drbg_ro_sanity;
1116#endif
1117#ifndef NO_RSA
1118            typeof(wolfCrypt_FIPS_RSA_sanity) *wolfCrypt_FIPS_RSA_sanity;
1119            typeof(wolfCrypt_FIPS_rsa_ro_sanity) *wolfCrypt_FIPS_rsa_ro_sanity;
1120#endif
1121#ifndef NO_SHA
1122            typeof(wolfCrypt_FIPS_SHA_sanity) *wolfCrypt_FIPS_SHA_sanity;
1123            typeof(wolfCrypt_FIPS_sha_ro_sanity) *wolfCrypt_FIPS_sha_ro_sanity;
1124#endif
1125#ifndef NO_SHA256
1126            typeof(wolfCrypt_FIPS_SHA256_sanity) *wolfCrypt_FIPS_SHA256_sanity;
1127            typeof(wolfCrypt_FIPS_sha256_ro_sanity) *wolfCrypt_FIPS_sha256_ro_sanity;
1128#endif
1129#ifdef WOLFSSL_SHA512
1130            typeof(wolfCrypt_FIPS_SHA512_sanity) *wolfCrypt_FIPS_SHA512_sanity;
1131            typeof(wolfCrypt_FIPS_sha512_ro_sanity) *wolfCrypt_FIPS_sha512_ro_sanity;
1132#endif
1133#ifdef WOLFSSL_SHA3
1134            typeof(wolfCrypt_FIPS_SHA3_sanity) *wolfCrypt_FIPS_SHA3_sanity;
1135            typeof(wolfCrypt_FIPS_sha3_ro_sanity) *wolfCrypt_FIPS_sha3_ro_sanity;
1136#endif
1137            typeof(wolfCrypt_FIPS_FT_sanity) *wolfCrypt_FIPS_FT_sanity;
1138            typeof(wolfCrypt_FIPS_ft_ro_sanity) *wolfCrypt_FIPS_ft_ro_sanity;
1139            typeof(wolfCrypt_FIPS_f_ro_sanity) *wolfCrypt_FIPS_f_ro_sanity;
1140            typeof(wc_RunAllCast_fips) *wc_RunAllCast_fips;
1141        #endif
1142        #endif
1143
1144        #if !defined(WOLFCRYPT_ONLY) && !defined(NO_CERTS)
1145        typeof(GetCA) *GetCA;
1146        #ifndef NO_SKID
1147        typeof(GetCAByName) *GetCAByName;
1148        #ifdef HAVE_OCSP
1149        typeof(GetCAByKeyHash) *GetCAByKeyHash;
1150        #endif /* HAVE_OCSP */
1151        #endif /* NO_SKID */
1152        #ifdef WOLFSSL_AKID_NAME
1153        typeof(GetCAByAKID) *GetCAByAKID;
1154        #endif /* WOLFSSL_AKID_NAME */
1155
1156        #if defined(OPENSSL_EXTRA) || defined(OPENSSL_EXTRA_X509_SMALL)
1157        typeof(wolfSSL_X509_NAME_add_entry_by_NID) *wolfSSL_X509_NAME_add_entry_by_NID;
1158        typeof(wolfSSL_X509_NAME_free) *wolfSSL_X509_NAME_free;
1159        typeof(wolfSSL_X509_NAME_new_ex) *wolfSSL_X509_NAME_new_ex;
1160        #endif /* OPENSSL_EXTRA || OPENSSL_EXTRA_X509_SMALL */
1161
1162        #endif /* !WOLFCRYPT_ONLY && !NO_CERTS */
1163
1164        typeof(dump_stack) *dump_stack;
1165
1166        #ifdef CONFIG_ARM64
1167        #ifndef CONFIG_ARCH_TEGRA
1168        #ifdef WC_CONTAINERIZE_THIS
1169            /* alt_cb_patch_nops and queued_spin_lock_slowpath are defined early
1170             * to allow shimming in system headers, but now we need the native
1171             * ones.
1172             */
1173            #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)
1174            #undef alt_cb_patch_nops
1175            typeof(my__alt_cb_patch_nops) *alt_cb_patch_nops;
1176            #endif /* LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0) */
1177            #undef queued_spin_lock_slowpath
1178            typeof(my__queued_spin_lock_slowpath) *queued_spin_lock_slowpath;
1179        #else
1180            #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0)
1181            typeof(alt_cb_patch_nops) *alt_cb_patch_nops;
1182            #endif
1183            typeof(queued_spin_lock_slowpath) *queued_spin_lock_slowpath;
1184        #endif
1185        #endif
1186        #endif
1187
1188        typeof(preempt_count) *preempt_count;
1189        #ifndef _raw_spin_lock_irqsave
1190            typeof(_raw_spin_lock_irqsave) *_raw_spin_lock_irqsave;
1191        #endif
1192        #ifndef _raw_spin_trylock
1193            typeof(_raw_spin_trylock) *_raw_spin_trylock;
1194        #endif
1195        #ifndef _raw_spin_unlock_irqrestore
1196            typeof(_raw_spin_unlock_irqrestore) *_raw_spin_unlock_irqrestore;
1197        #endif
1198        typeof(_cond_resched) *_cond_resched;
1199        #ifndef WOLFSSL_LINUXKM_USE_MUTEXES
1200        typeof(wc_lkm_LockMutex) *wc_lkm_LockMutex;
1201        #endif
1202
1203        typeof(wc_linuxkm_sig_ignore_begin) *wc_linuxkm_sig_ignore_begin;
1204        typeof(wc_linuxkm_sig_ignore_end) *wc_linuxkm_sig_ignore_end;
1205        typeof(wc_linuxkm_check_for_intr_signals) *wc_linuxkm_check_for_intr_signals;
1206        typeof(wc_linuxkm_relax_long_loop) *wc_linuxkm_relax_long_loop;
1207
1208        #ifdef CONFIG_KASAN
1209            typeof(kasan_disable_current) *kasan_disable_current;
1210            typeof(kasan_enable_current) *kasan_enable_current;
1211        #endif
1212
1213        const void *_last_slot;
1214    };
1215
1216    extern const struct wolfssl_linuxkm_pie_redirect_table *wolfssl_linuxkm_get_pie_redirect_table(void);
1217    extern struct wolfssl_linuxkm_pie_redirect_table wolfssl_linuxkm_pie_redirect_table;
1218
1219
1220    #if defined(WC_PIE_INDIRECT_SYM)
1221        /* keep user-supplied override definition. */
1222    #elif defined(WC_PIE_INDIRECT_SYM_BY_FUNC_ONLY) || \
1223        defined(WC_PIE_INDIRECT_SYM_BY_DIRECT_TABLE_READ)
1224        /* keep user-supplied override method. */
1225    #elif defined(CONFIG_X86)
1226        #define WC_PIE_INDIRECT_SYM_BY_DIRECT_TABLE_READ
1227    #elif defined(CONFIG_ARM64)
1228        /* direct access to wolfssl_linuxkm_pie_redirect_table.x on aarch64
1229         * produces GOT relocations, e.g. R_AARCH64_LD64_GOT_LO12_NC.
1230         */
1231        #define WC_PIE_INDIRECT_SYM_BY_FUNC_ONLY
1232    #else
1233        /* for other archs, by default use the safe way. */
1234        #define WC_PIE_INDIRECT_SYM_BY_FUNC_ONLY
1235    #endif
1236
1237    #if defined(WC_PIE_INDIRECT_SYM)
1238        /* keep user-supplied override definition. */
1239    #elif defined(WC_PIE_INDIRECT_SYM_BY_FUNC_ONLY)
1240        static const struct wolfssl_linuxkm_pie_redirect_table *wolfssl_linuxkm_pie_redirect_table_local_ptr = NULL;
1241        static inline const struct wolfssl_linuxkm_pie_redirect_table *wolfssl_linuxkm_get_pie_redirect_table_local(void) {
1242            if (wolfssl_linuxkm_pie_redirect_table_local_ptr == NULL)
1243                wolfssl_linuxkm_pie_redirect_table_local_ptr = wolfssl_linuxkm_get_pie_redirect_table();
1244            return wolfssl_linuxkm_pie_redirect_table_local_ptr;
1245        }
1246        #define WC_PIE_INDIRECT_SYM(x) (wolfssl_linuxkm_get_pie_redirect_table_local()->x)
1247    #elif defined(WC_PIE_INDIRECT_SYM_BY_DIRECT_TABLE_READ)
1248        #define WC_PIE_INDIRECT_SYM(x) (wolfssl_linuxkm_pie_redirect_table.x)
1249    #else
1250        #error no WC_PIE_INDIRECT_SYM method defined.
1251    #endif
1252
1253    #ifdef WC_CONTAINERIZE_THIS
1254
1255    #ifdef HAVE_FIPS
1256        #define wc_linuxkm_normalize_relocations \
1257            WC_PIE_INDIRECT_SYM(wc_linuxkm_normalize_relocations)
1258        #define wc_linuxkm_normalize_relocations_noresize \
1259            WC_PIE_INDIRECT_SYM(wc_linuxkm_normalize_relocations_noresize)
1260    #endif
1261
1262    #ifndef __ARCH_MEMCMP_NO_REDIRECT
1263        #define memcmp WC_PIE_INDIRECT_SYM(memcmp)
1264    #endif
1265    #ifndef __ARCH_MEMCPY_NO_REDIRECT
1266        #define memcpy WC_PIE_INDIRECT_SYM(memcpy)
1267    #endif
1268    #ifndef __ARCH_MEMSET_NO_REDIRECT
1269        #define memset WC_PIE_INDIRECT_SYM(memset)
1270    #endif
1271    #ifndef __ARCH_MEMMOVE_NO_REDIRECT
1272        #define memmove WC_PIE_INDIRECT_SYM(memmove)
1273    #endif
1274    #ifndef __ARCH_STRCMP_NO_REDIRECT
1275        #define strcmp WC_PIE_INDIRECT_SYM(strcmp)
1276    #endif
1277    #ifndef __ARCH_STRNCMP_NO_REDIRECT
1278        #define strncmp WC_PIE_INDIRECT_SYM(strncmp)
1279    #endif
1280    #ifndef __ARCH_STRCASECMP_NO_REDIRECT
1281        #define strcasecmp WC_PIE_INDIRECT_SYM(strcasecmp)
1282    #endif
1283    #ifndef __ARCH_STRNCASECMP_NO_REDIRECT
1284        #define strncasecmp WC_PIE_INDIRECT_SYM(strncasecmp)
1285    #endif
1286    #ifndef __ARCH_STRLEN_NO_REDIRECT
1287        #define strlen WC_PIE_INDIRECT_SYM(strlen)
1288    #endif
1289    #ifndef __ARCH_STRSTR_NO_REDIRECT
1290        #define strstr WC_PIE_INDIRECT_SYM(strstr)
1291    #endif
1292    #ifndef __ARCH_STRNCPY_NO_REDIRECT
1293        #define strncpy WC_PIE_INDIRECT_SYM(strncpy)
1294    #endif
1295    #ifndef __ARCH_STRNCAT_NO_REDIRECT
1296        #define strncat WC_PIE_INDIRECT_SYM(strncat)
1297    #endif
1298    #define kstrtoll WC_PIE_INDIRECT_SYM(kstrtoll)
1299
1300    #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0)) || \
1301        (defined(RHEL_MAJOR) && \
1302         ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
1303        #define _printk WC_PIE_INDIRECT_SYM(_printk)
1304    #else
1305        #define printk WC_PIE_INDIRECT_SYM(printk)
1306    #endif
1307
1308    #ifdef CONFIG_FORTIFY_SOURCE
1309        #define __warn_printk WC_PIE_INDIRECT_SYM(__warn_printk)
1310    #endif
1311
1312    #define snprintf WC_PIE_INDIRECT_SYM(snprintf)
1313
1314    #define _ctype WC_PIE_INDIRECT_SYM(_ctype)
1315
1316#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 18, 0)
1317    /* see include/linux/alloc_tag.h and include/linux/slab.h */
1318    #define kmalloc_noprof WC_PIE_INDIRECT_SYM(kmalloc_noprof)
1319    #define krealloc_node_align_noprof WC_PIE_INDIRECT_SYM(krealloc_node_align_noprof)
1320    #define kzalloc_noprof WC_PIE_INDIRECT_SYM(kzalloc_noprof)
1321    #define __kvmalloc_node_noprof WC_PIE_INDIRECT_SYM(__kvmalloc_node_noprof)
1322    #define __kmalloc_cache_noprof WC_PIE_INDIRECT_SYM(__kmalloc_cache_noprof)
1323    #ifdef HAVE_KVREALLOC
1324        #define kvrealloc_node_align_noprof WC_PIE_INDIRECT_SYM(kvrealloc_node_align_noprof)
1325    #endif
1326#elif LINUX_VERSION_CODE >= KERNEL_VERSION(6, 11, 0)
1327    /* see include/linux/alloc_tag.h and include/linux/slab.h */
1328    #define kmalloc_noprof WC_PIE_INDIRECT_SYM(kmalloc_noprof)
1329    #define krealloc_noprof WC_PIE_INDIRECT_SYM(krealloc_noprof)
1330    #define kzalloc_noprof WC_PIE_INDIRECT_SYM(kzalloc_noprof)
1331    #define __kvmalloc_node_noprof WC_PIE_INDIRECT_SYM(__kvmalloc_node_noprof)
1332    #define __kmalloc_cache_noprof WC_PIE_INDIRECT_SYM(__kmalloc_cache_noprof)
1333    #ifdef HAVE_KVREALLOC
1334        #define kvrealloc_noprof WC_PIE_INDIRECT_SYM(kvrealloc_noprof)
1335    #endif
1336#elif LINUX_VERSION_CODE >= KERNEL_VERSION(6, 10, 0)
1337    /* see include/linux/alloc_tag.h and include/linux/slab.h */
1338    #define kmalloc_noprof WC_PIE_INDIRECT_SYM(kmalloc_noprof)
1339    #define krealloc_noprof WC_PIE_INDIRECT_SYM(krealloc_noprof)
1340    #define kzalloc_noprof WC_PIE_INDIRECT_SYM(kzalloc_noprof)
1341    #define kvmalloc_node_noprof WC_PIE_INDIRECT_SYM(kvmalloc_node_noprof)
1342    #define kmalloc_trace_noprof WC_PIE_INDIRECT_SYM(kmalloc_trace_noprof)
1343    #ifdef HAVE_KVREALLOC
1344        #define kvrealloc_noprof WC_PIE_INDIRECT_SYM(kvrealloc_noprof)
1345    #endif
1346#else /* <6.10.0 */
1347    #define kmalloc WC_PIE_INDIRECT_SYM(kmalloc)
1348    #define krealloc WC_PIE_INDIRECT_SYM(krealloc)
1349    #define kzalloc(size, flags) kmalloc(size, (flags) | __GFP_ZERO)
1350    #ifdef HAVE_KVMALLOC
1351        #define kvmalloc_node WC_PIE_INDIRECT_SYM(kvmalloc_node)
1352    #endif
1353    #ifdef HAVE_KVREALLOC
1354        #define kvrealloc WC_PIE_INDIRECT_SYM(kvrealloc)
1355    #endif
1356    #if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 1, 0) || \
1357        (defined(RHEL_MAJOR) &&                                    \
1358         ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
1359        #define kmalloc_trace WC_PIE_INDIRECT_SYM(kmalloc_trace)
1360    #else
1361        #define kmem_cache_alloc_trace WC_PIE_INDIRECT_SYM(kmem_cache_alloc_trace)
1362        #define kmalloc_order_trace WC_PIE_INDIRECT_SYM(kmalloc_order_trace)
1363    #endif
1364#endif /* <6.10.0 */
1365
1366    #define kfree WC_PIE_INDIRECT_SYM(kfree)
1367    #ifdef HAVE_KVMALLOC
1368        #define kvfree WC_PIE_INDIRECT_SYM(kvfree)
1369    #endif
1370    #define ksize WC_PIE_INDIRECT_SYM(ksize)
1371
1372#ifndef LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT
1373    #define get_random_bytes WC_PIE_INDIRECT_SYM(get_random_bytes)
1374#endif
1375    #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
1376        #define getnstimeofday WC_PIE_INDIRECT_SYM(getnstimeofday)
1377    #elif LINUX_VERSION_CODE < KERNEL_VERSION(5, 0, 0)
1378        #define current_kernel_time64 WC_PIE_INDIRECT_SYM(current_kernel_time64)
1379    #else
1380        #define ktime_get_coarse_real_ts64 WC_PIE_INDIRECT_SYM(ktime_get_coarse_real_ts64)
1381    #endif
1382
1383    #undef get_current
1384    #define get_current WC_PIE_INDIRECT_SYM(get_current)
1385
1386    #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS) && defined(CONFIG_X86)
1387        #define allocate_wolfcrypt_linuxkm_fpu_states WC_PIE_INDIRECT_SYM(allocate_wolfcrypt_linuxkm_fpu_states)
1388        #define wc_can_save_vector_registers_x86 WC_PIE_INDIRECT_SYM(wc_can_save_vector_registers_x86)
1389        #define free_wolfcrypt_linuxkm_fpu_states WC_PIE_INDIRECT_SYM(free_wolfcrypt_linuxkm_fpu_states)
1390        #define wc_restore_vector_registers_x86 WC_PIE_INDIRECT_SYM(wc_restore_vector_registers_x86)
1391        #define wc_save_vector_registers_x86 WC_PIE_INDIRECT_SYM(wc_save_vector_registers_x86)
1392    #elif defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)
1393        #error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture.
1394    #endif /* WOLFSSL_USE_SAVE_VECTOR_REGISTERS */
1395
1396    #ifndef CONFIG_PREEMPT_RT
1397        #define __mutex_init WC_PIE_INDIRECT_SYM(__mutex_init)
1398    #else
1399        /* On RT kernels, __mutex_init is a macro pointing to __rt_mutex_init */
1400        #undef __mutex_init
1401        #define __rt_mutex_init WC_PIE_INDIRECT_SYM(__rt_mutex_init)
1402        #define __mutex_init(mutex, name, key) __rt_mutex_init(mutex, name, key)
1403        #define rt_mutex_base_init WC_PIE_INDIRECT_SYM(rt_mutex_base_init)
1404        #define rt_spin_lock WC_PIE_INDIRECT_SYM(rt_spin_lock)
1405        #define rt_spin_unlock WC_PIE_INDIRECT_SYM(rt_spin_unlock)
1406    #endif
1407    #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
1408        #define mutex_lock_nested WC_PIE_INDIRECT_SYM(mutex_lock_nested)
1409    #else
1410        #define mutex_lock WC_PIE_INDIRECT_SYM(mutex_lock)
1411    #endif
1412    #define mutex_unlock WC_PIE_INDIRECT_SYM(mutex_unlock)
1413    #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
1414        #define mutex_destroy WC_PIE_INDIRECT_SYM(mutex_destroy)
1415    #endif
1416
1417    /* per linux/ctype.h, tolower() and toupper() are macros bound to static inlines
1418     * that use macros that bring in the _ctype global.  for WC_CONTAINERIZE_THIS,
1419     * this needs to be masked out.
1420     */
1421    #undef tolower
1422    #undef toupper
1423    #define tolower(c) (islower(c) ? (c) : ((c) + ('a'-'A')))
1424    #define toupper(c) (isupper(c) ? (c) : ((c) - ('a'-'A')))
1425
1426    #if !defined(WOLFCRYPT_ONLY) && !defined(NO_CERTS)
1427        #define GetCA WC_PIE_INDIRECT_SYM(GetCA)
1428        #ifndef NO_SKID
1429            #define GetCAByName WC_PIE_INDIRECT_SYM(GetCAByName)
1430            #ifdef HAVE_OCSP
1431                #define GetCAByKeyHash WC_PIE_INDIRECT_SYM(GetCAByKeyHash)
1432            #endif /* HAVE_OCSP */
1433        #endif /* NO_SKID */
1434        #ifdef WOLFSSL_AKID_NAME
1435            #ifdef WOLFSSL_API_PREFIX_MAP
1436                #define wolfSSL_GetCAByAKID WC_PIE_INDIRECT_SYM(wolfSSL_GetCAByAKID)
1437            #else
1438                #define GetCAByAKID WC_PIE_INDIRECT_SYM(GetCAByAKID)
1439            #endif
1440        #endif
1441
1442        #if defined(OPENSSL_EXTRA) || defined(OPENSSL_EXTRA_X509_SMALL)
1443            #define wolfSSL_X509_NAME_add_entry_by_NID WC_PIE_INDIRECT_SYM(wolfSSL_X509_NAME_add_entry_by_NID)
1444            #define wolfSSL_X509_NAME_free WC_PIE_INDIRECT_SYM(wolfSSL_X509_NAME_free)
1445            #define wolfSSL_X509_NAME_new_ex WC_PIE_INDIRECT_SYM(wolfSSL_X509_NAME_new_ex)
1446        #endif /* OPENSSL_EXTRA || OPENSSL_EXTRA_X509_SMALL */
1447
1448    #endif /* !WOLFCRYPT_ONLY && !NO_CERTS */
1449
1450        #define dump_stack WC_PIE_INDIRECT_SYM(dump_stack)
1451
1452    #undef preempt_count /* just in case -- not a macro on x86. */
1453    #define preempt_count WC_PIE_INDIRECT_SYM(preempt_count)
1454
1455    #ifndef WOLFSSL_LINUXKM_USE_MUTEXES
1456        #ifndef _raw_spin_lock_irqsave
1457            #define _raw_spin_lock_irqsave WC_PIE_INDIRECT_SYM(_raw_spin_lock_irqsave)
1458        #endif
1459        #ifndef _raw_spin_trylock
1460            #define _raw_spin_trylock WC_PIE_INDIRECT_SYM(_raw_spin_trylock)
1461        #endif
1462        #ifndef _raw_spin_unlock_irqrestore
1463            #define _raw_spin_unlock_irqrestore WC_PIE_INDIRECT_SYM(_raw_spin_unlock_irqrestore)
1464        #endif
1465    #endif
1466
1467    #define _cond_resched WC_PIE_INDIRECT_SYM(_cond_resched)
1468
1469    /* this is defined in linux/spinlock.h as an inline that calls the unshimmed
1470     * raw_spin_unlock_irqrestore().  use a macro here to supersede it.
1471     * Note: On PREEMPT_RT kernels, spinlock_t doesn't have rlock member,
1472     * so we skip this redefinition and use the kernel's native implementation.
1473     */
1474    #ifndef CONFIG_PREEMPT_RT
1475        #define spin_unlock_irqrestore(lock, flags) raw_spin_unlock_irqrestore(&((lock)->rlock), flags)
1476    #else
1477        /* Undo internal wolfSSL PIE macro rewriting */
1478        #ifdef rt_spin_unlock
1479        #undef rt_spin_unlock
1480        #endif
1481        #ifdef rt_spin_lock
1482        #undef rt_spin_lock
1483        #endif
1484        static inline int wolfssl_spin_unlock_irqrestore_rt(spinlock_t *lock,
1485                                                             unsigned long flags)
1486        {
1487            (void)flags; /* rt_spin_unlock ignores flags */
1488            WC_PIE_INDIRECT_SYM(rt_spin_unlock)(lock);
1489            return 0;
1490        }
1491
1492        #undef spin_unlock_irqrestore
1493        #define spin_unlock_irqrestore(lock, flags) \
1494        wolfssl_spin_unlock_irqrestore_rt((lock), (flags))
1495    #endif
1496
1497    #define wc_linuxkm_sig_ignore_begin WC_PIE_INDIRECT_SYM(wc_linuxkm_sig_ignore_begin);
1498    #define wc_linuxkm_sig_ignore_end WC_PIE_INDIRECT_SYM(wc_linuxkm_sig_ignore_end);
1499    #define wc_linuxkm_check_for_intr_signals WC_PIE_INDIRECT_SYM(wc_linuxkm_check_for_intr_signals)
1500    #define wc_linuxkm_relax_long_loop WC_PIE_INDIRECT_SYM(wc_linuxkm_relax_long_loop)
1501
1502    #ifdef CONFIG_KASAN
1503        #define kasan_disable_current WC_PIE_INDIRECT_SYM(kasan_disable_current)
1504        #define kasan_enable_current WC_PIE_INDIRECT_SYM(kasan_enable_current)
1505    #endif
1506
1507    #endif /* WC_CONTAINERIZE_THIS */
1508
1509    #endif /* WC_SYM_RELOC_TABLES */
1510
1511#if defined(WOLFSSL_KERNEL_STACK_DEBUG) || defined(WC_LINUXKM_STACK_DEBUG)
1512
1513    #ifndef CONFIG_THREAD_INFO_IN_TASK
1514        #error WC_LINUXKM_STACK_DEBUG requires CONFIG_THREAD_INFO_IN_TASK
1515    #endif
1516    #ifdef CONFIG_STACK_GROWSUP
1517        #error WC_LINUXKM_STACK_DEBUG requires !CONFIG_STACK_GROWSUP
1518    #endif
1519
1520    static __always_inline unsigned long wc_linuxkm_stack_bottom(void) {
1521        void *ret = task_stack_page(get_current());
1522        return (unsigned long)(uintptr_t)ret;
1523    }
1524
1525    static __always_inline unsigned long wc_linuxkm_stack_top(void) {
1526        return wc_linuxkm_stack_bottom() + THREAD_SIZE;
1527    }
1528
1529    #if defined(CONFIG_X86)
1530
1531    static __always_inline unsigned long wc_linuxkm_stack_current(void) {
1532        unsigned long rsp;
1533
1534        asm volatile("mov %%rsp, %0" : "=r" (rsp));
1535        return wc_linuxkm_stack_top() - rsp;
1536    }
1537
1538    static __always_inline unsigned long wc_linuxkm_stack_left(void) {
1539        unsigned long rsp;
1540        asm volatile("mov %%rsp, %0" : "=r" (rsp));
1541        return rsp - wc_linuxkm_stack_bottom();
1542    }
1543
1544    #define WC_LINUXKM_HAVE_STACK_DEBUG
1545
1546    #elif defined(CONFIG_ARM64)
1547
1548    static __always_inline unsigned long wc_linuxkm_stack_current(void) {
1549        unsigned long sp;
1550        asm volatile("mov %0, sp" : "=r" (sp));
1551        return wc_linuxkm_stack_top() - sp;
1552    }
1553
1554    static __always_inline unsigned long wc_linuxkm_stack_left(void) {
1555        unsigned long sp;
1556        asm volatile("mov %0, sp" : "=r" (sp));
1557        return sp - wc_linuxkm_stack_bottom();
1558    }
1559
1560    #define WC_LINUXKM_HAVE_STACK_DEBUG
1561
1562    #elif defined(CONFIG_ARM)
1563
1564    static __always_inline unsigned long wc_linuxkm_stack_current(void) {
1565        unsigned long sp;
1566        asm volatile("mov %0, sp" : "=r" (sp));
1567        return wc_linuxkm_stack_top() - sp;
1568    }
1569
1570    static __always_inline unsigned long wc_linuxkm_stack_left(void) {
1571        unsigned long sp;
1572        asm volatile("mov %0, sp" : "=r" (sp));
1573        return sp - wc_linuxkm_stack_bottom();
1574    }
1575
1576    #define WC_LINUXKM_HAVE_STACK_DEBUG
1577
1578    #endif /* CONFIG_ARM */
1579
1580    #ifndef WC_LINUXKM_HAVE_STACK_DEBUG
1581        #error WC_LINUXKM_STACK_DEBUG implementation missing for target.
1582    #endif
1583
1584    /* An unsigned long STACK_END_MAGIC is stored at the bottom of the stack.
1585     * Additionally, though the kernel stack doesn't have a red zone, it
1586     * nonetheless uses some bytes below the current stack pointer and mayhem
1587     * ensues immediately if it's overwritten.
1588     */
1589    #ifndef WC_KERNEL_STACK_MARGIN_BOTTOM
1590        #define WC_KERNEL_STACK_MARGIN_BOTTOM sizeof(unsigned long)
1591    #endif
1592    #ifndef WC_KERNEL_STACK_MARGIN_TOP
1593        #define WC_KERNEL_STACK_MARGIN_TOP 8
1594    #endif
1595
1596    static __always_inline void wc_linuxkm_stack_hwm_prepare(unsigned char sentinel) {
1597        unsigned long s = wc_linuxkm_stack_bottom();
1598        unsigned long z;
1599        unsigned long flags;
1600
1601        if (*(unsigned long *)s != STACK_END_MAGIC)
1602            pr_err("ERROR: bottom of stack is not STACK_END_MAGIC.\n");
1603
1604        local_irq_save(flags);
1605        kasan_disable_current();
1606        z = wc_linuxkm_stack_left();
1607        if (z > WC_KERNEL_STACK_MARGIN_BOTTOM + WC_KERNEL_STACK_MARGIN_TOP)
1608            memset((void *)(s + WC_KERNEL_STACK_MARGIN_BOTTOM), sentinel,
1609                   z - (WC_KERNEL_STACK_MARGIN_BOTTOM + WC_KERNEL_STACK_MARGIN_TOP));
1610        kasan_enable_current();
1611        local_irq_restore(flags);
1612        if (z <= WC_KERNEL_STACK_MARGIN_BOTTOM + WC_KERNEL_STACK_MARGIN_TOP)
1613            pr_err("ERROR: wc_linuxkm_stack_hwm_prepare() called with only %lu bytes of stack left, "
1614                   "versus margin %zu.\n", z, WC_KERNEL_STACK_MARGIN_BOTTOM + WC_KERNEL_STACK_MARGIN_TOP);
1615    }
1616    static __always_inline unsigned long wc_linuxkm_stack_hwm_measure_rel(unsigned char sentinel) {
1617        unsigned long s = wc_linuxkm_stack_bottom();
1618        unsigned long z = wc_linuxkm_stack_left();
1619        unsigned char *i;
1620        if (z <= WC_KERNEL_STACK_MARGIN_BOTTOM + WC_KERNEL_STACK_MARGIN_TOP)
1621            return (unsigned long)-1;
1622        kasan_disable_current();
1623        for (i = (unsigned char *)s + WC_KERNEL_STACK_MARGIN_BOTTOM;
1624             i < ((unsigned char *)s + z) && (*i == sentinel);
1625             ++i);
1626        kasan_enable_current();
1627        return z - ((unsigned long)i - s);
1628    }
1629    static __always_inline unsigned long wc_linuxkm_stack_hwm_measure_total(unsigned char sentinel) {
1630        unsigned long rel = wc_linuxkm_stack_hwm_measure_rel(sentinel);
1631        if (rel == (unsigned long)-1)
1632            return rel;
1633        else
1634            return rel + wc_linuxkm_stack_current();
1635    }
1636
1637#endif /* WC_LINUXKM_STACK_DEBUG */
1638
1639    /* remove this multifariously conflicting macro, picked up from
1640     * Linux arch/<arch>/include/asm/current.h.
1641     */
1642    #ifndef WOLFSSL_LINUXKM_NEED_LINUX_CURRENT
1643        #undef current
1644    #endif
1645
1646    /* min() and max() in linux/kernel.h over-aggressively type-check, producing
1647     * myriad spurious -Werrors throughout the codebase.
1648     */
1649    #undef min
1650    #undef max
1651
1652    /* work around namespace conflict between wolfssl/internal.h (enum HandShakeType)
1653     * and linux/key.h (extern int()).
1654     */
1655    #define key_update wc_key_update
1656
1657    #if (LINUX_VERSION_CODE >= KERNEL_VERSION(5, 15, 0)) || \
1658        (defined(RHEL_MAJOR) && \
1659         ((RHEL_MAJOR > 9) || ((RHEL_MAJOR == 9) && (RHEL_MINOR >= 5))))
1660        #define wc_km_printf(format, args...) _printk(KERN_INFO "wolfssl: %s(): " format, __func__, ## args)
1661    #else
1662        #define wc_km_printf(format, args...) printk(KERN_INFO "wolfssl: %s(): " format, __func__, ## args)
1663    #endif
1664    #define printf(...) wc_km_printf(__VA_ARGS__)
1665
1666    #ifdef HAVE_FIPS
1667        extern void fipsEntry(void);
1668    #endif
1669
1670    /* suppress false-positive "writing 1 byte into a region of size 0" warnings
1671     * building old kernels with new gcc:
1672     */
1673    #if LINUX_VERSION_CODE < KERNEL_VERSION(4, 0, 0)
1674    _Pragma("GCC diagnostic ignored \"-Wstringop-overflow\"");
1675    #endif
1676
1677    /* includes are all above, with incompatible warnings masked out. */
1678    #if LINUX_VERSION_CODE < KERNEL_VERSION(5, 5, 0)
1679    typedef __kernel_time_t time_t;
1680    #else
1681    typedef __kernel_time64_t time_t;
1682    #endif
1683    extern time_t time(time_t * timer);
1684    #define XTIME time
1685    #define WOLFSSL_GMTIME
1686    #define XGMTIME(c, t) gmtime(c)
1687    #define NO_TIMEVAL 1
1688
1689    #endif /* BUILDING_WOLFSSL */
1690
1691    #if !defined(BUILDING_WOLFSSL)
1692        /* some caller code needs these. */
1693        #if defined(WOLFSSL_USE_SAVE_VECTOR_REGISTERS)
1694            #if defined(CONFIG_X86)
1695                WOLFSSL_API __must_check int wc_can_save_vector_registers_x86(void);
1696                WOLFSSL_API __must_check int wc_save_vector_registers_x86(enum wc_svr_flags flags);
1697                WOLFSSL_API void wc_restore_vector_registers_x86(enum wc_svr_flags flags);
1698                #ifndef DISABLE_VECTOR_REGISTERS
1699                    #define DISABLE_VECTOR_REGISTERS() wc_save_vector_registers_x86(WC_SVR_FLAG_INHIBIT)
1700                #endif
1701                #ifndef REENABLE_VECTOR_REGISTERS
1702                    #define REENABLE_VECTOR_REGISTERS() wc_restore_vector_registers_x86(WC_SVR_FLAG_INHIBIT)
1703                #endif
1704            #else /* !CONFIG_X86 */
1705                #error WOLFSSL_USE_SAVE_VECTOR_REGISTERS is set for an unimplemented architecture.
1706            #endif /* !CONFIG_X86 */
1707        #endif /* WOLFSSL_USE_SAVE_VECTOR_REGISTERS */
1708    #endif /* !BUILDING_WOLFSSL */
1709
1710    /* Copied from wc_port.h */
1711    #if defined(HAVE_FIPS) && !defined(WOLFSSL_API_PREFIX_MAP)
1712        /* For FIPS keep the function names the same */
1713        #define wc_InitMutex   InitMutex
1714        #define wc_FreeMutex   FreeMutex
1715        #define wc_LockMutex   LockMutex
1716        #define wc_UnLockMutex UnLockMutex
1717    #endif /* HAVE_FIPS */
1718
1719    #ifdef WOLFSSL_LINUXKM_USE_MUTEXES
1720        #define WC_MUTEX_OPS_INLINE
1721
1722        #ifdef LINUXKM_LKCAPI_REGISTER
1723            /* must use spin locks when registering implementations with the
1724             * kernel, because mutexes are forbidden when calling with nonzero
1725             * irq_count().
1726             */
1727            #error WOLFSSL_LINUXKM_USE_MUTEXES is incompatible with LINUXKM_LKCAPI_REGISTER.
1728        #endif
1729
1730        /* if BUILDING_WOLFSSL, mutex.h will have already been included
1731         * recursively above, with the bevy of warnings suppressed, and the
1732         * below include will be a redundant no-op.
1733         */
1734        #include <linux/mutex.h>
1735        typedef struct mutex wolfSSL_Mutex;
1736        #define WOLFSSL_MUTEX_INITIALIZER(lockname) __MUTEX_INITIALIZER(lockname)
1737
1738        /* Linux kernel mutex routines are voids, alas. */
1739
1740        static inline int wc_InitMutex(wolfSSL_Mutex* m)
1741        {
1742            mutex_init(m);
1743            return 0;
1744        }
1745
1746        static inline int wc_FreeMutex(wolfSSL_Mutex* m)
1747        {
1748            mutex_destroy(m);
1749            return 0;
1750        }
1751
1752        static __must_check inline int wc_LockMutex(wolfSSL_Mutex* m)
1753        {
1754            if (in_nmi() || hardirq_count() || in_softirq())
1755                return -1;
1756            mutex_lock(m);
1757            return 0;
1758        }
1759
1760        static inline int wc_UnLockMutex(wolfSSL_Mutex* m)
1761        {
1762            mutex_unlock(m);
1763            return 0;
1764        }
1765    #else
1766        #define WC_MUTEX_OPS_INLINE
1767
1768        /* if BUILDING_WOLFSSL, spinlock.h will have already been included
1769         * recursively above, with the bevy of warnings suppressed, and the
1770         * below include will be a redundant no-op.
1771         */
1772        #include <linux/spinlock.h>
1773
1774        typedef struct wolfSSL_Mutex {
1775            spinlock_t lock;
1776            unsigned long irq_flags;
1777        } wolfSSL_Mutex;
1778
1779        #define WOLFSSL_MUTEX_INITIALIZER(lockname) { .lock =__SPIN_LOCK_UNLOCKED(lockname), .irq_flags = 0 }
1780
1781        static __always_inline int wc_InitMutex(wolfSSL_Mutex* m)
1782        {
1783        /* Tegra vendor kernels do not support assignment of __SPIN_LOCK_UNLOCKED() */
1784        # ifndef CONFIG_ARCH_TEGRA
1785            m->lock = __SPIN_LOCK_UNLOCKED(m);
1786        # else
1787            spin_lock_init(&m->lock);
1788        #endif
1789            m->irq_flags = 0;
1790
1791            return 0;
1792        }
1793
1794        static __always_inline int wc_FreeMutex(wolfSSL_Mutex* m)
1795        {
1796            (void)m;
1797            return 0;
1798        }
1799
1800        #ifdef WC_CONTAINERIZE_THIS
1801        /* wc_lkm_LockMutex() can't be used inline in WC_CONTAINERIZE_THIS objects, due to
1802         * direct access to pv_ops.
1803         */
1804        static __must_check __always_inline int wc_LockMutex(wolfSSL_Mutex *m)
1805        {
1806            return WC_PIE_INDIRECT_SYM(wc_lkm_LockMutex)(m);
1807        }
1808
1809        #else /* !WC_CONTAINERIZE_THIS */
1810
1811        static __must_check __always_inline int wc_LockMutex(wolfSSL_Mutex *m)
1812        {
1813            return wc_lkm_LockMutex(m);
1814        }
1815
1816        #endif /* !WC_CONTAINERIZE_THIS */
1817
1818        static __always_inline int wc_UnLockMutex(wolfSSL_Mutex* m)
1819        {
1820            spin_unlock_irqrestore(&m->lock, m->irq_flags);
1821            return 0;
1822        }
1823
1824    #endif
1825
1826    #ifdef LINUXKM_LKCAPI_REGISTER_HASH_DRBG_DEFAULT
1827        struct crypto_rng;
1828        WOLFSSL_API int wc_linux_kernel_rng_is_wolfcrypt(struct crypto_rng *rng);
1829    #endif
1830
1831    /* Undo copied defines from wc_port.h, to avoid redefinition warnings. */
1832    #ifdef HAVE_FIPS
1833    #undef wc_InitMutex
1834    #undef wc_FreeMutex
1835    #undef wc_LockMutex
1836    #undef wc_UnLockMutex
1837    #endif /* HAVE_FIPS */
1838
1839    /* prevent gcc's mm_malloc.h from being included, since it unconditionally
1840     * includes stdlib.h, which is kernel-incompatible.
1841     */
1842    #define _MM_MALLOC_H_INCLUDED
1843
1844    #ifndef BUILDING_WOLFSSL
1845        #include <linux/slab.h>
1846        #if defined(USE_KVMALLOC) && (LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0))
1847            #include <linux/mm.h>
1848        #endif
1849    #endif
1850
1851    /* fun fact: since linux commit 59bb47985c, kmalloc with power-of-2 size is
1852     * aligned to the size.
1853     */
1854    #define WC_LINUXKM_ROUND_UP_P_OF_2(x) (                                \
1855    {                                                                      \
1856        size_t _alloc_sz = (x);                                            \
1857        if ((_alloc_sz < 8192) && (_alloc_sz != 0))                        \
1858          _alloc_sz = 1UL <<                                               \
1859              ((sizeof(_alloc_sz) * 8UL) - __builtin_clzl(_alloc_sz - 1)); \
1860        _alloc_sz;                                                         \
1861    })
1862    #ifdef USE_KVMALLOC
1863        #define malloc(size) kvmalloc_node(WC_LINUXKM_ROUND_UP_P_OF_2(size), (preempt_count() == 0 ? GFP_KERNEL : GFP_ATOMIC), NUMA_NO_NODE)
1864        #define free(ptr) kvfree(ptr)
1865        #ifdef USE_KVREALLOC
1866            #define realloc(ptr, newsize) kvrealloc(ptr, WC_LINUXKM_ROUND_UP_P_OF_2(newsize), (preempt_count() == 0 ? GFP_KERNEL : GFP_ATOMIC))
1867        #else
1868            #define realloc(ptr, newsize) ((void)(ptr), (void)(newsize), NULL)
1869        #endif
1870    #else
1871        #define malloc(size) kmalloc(WC_LINUXKM_ROUND_UP_P_OF_2(size), (preempt_count() == 0 ? GFP_KERNEL : GFP_ATOMIC))
1872        #define free(ptr) kfree(ptr)
1873        #define realloc(ptr, newsize) krealloc(ptr, WC_LINUXKM_ROUND_UP_P_OF_2(newsize), (preempt_count() == 0 ? GFP_KERNEL : GFP_ATOMIC))
1874    #endif
1875
1876    #ifndef static_assert
1877        #define static_assert(expr, ...) __static_assert(expr, ##__VA_ARGS__, #expr)
1878        #define __static_assert(expr, msg, ...) _Static_assert(expr, msg)
1879    #endif
1880
1881#ifdef WOLFSSL_TRACK_MEMORY
1882    #define XMALLOC(s, h, t)     ({(void)(h); (void)(t); wolfSSL_Malloc(s);})
1883    #ifdef WOLFSSL_XFREE_NO_NULLNESS_CHECK
1884        #define XFREE(p, h, t)       ({(void)(h); (void)(t); wolfSSL_Free(p);})
1885    #else
1886        #define XFREE(p, h, t)       ({void* _xp; (void)(h); _xp = (p); if(_xp) wolfSSL_Free(_xp);})
1887    #endif
1888    #define XREALLOC(p, n, h, t) ({(void)(h); (void)(t); wolfSSL_Realloc(p, n);})
1889#else
1890    #if !defined(XMALLOC_USER) && !defined(XMALLOC_OVERRIDE)
1891        #define XMALLOC(s, h, t)     ({(void)(h); (void)(t); malloc(s);})
1892        #ifdef WOLFSSL_XFREE_NO_NULLNESS_CHECK
1893            #define XFREE(p, h, t)       ({(void)(h); (void)(t); free(p);})
1894        #else
1895            #define XFREE(p, h, t)       ({void* _xp; (void)(h); (void)(t); _xp = (p); if(_xp) free(_xp);})
1896        #endif
1897        #if defined(USE_KVREALLOC) || !defined(USE_KVMALLOC)
1898            #define XREALLOC(p, n, h, t) ({(void)(h); (void)(t); realloc(p, n);})
1899        #endif
1900    #endif /* !XMALLOC_USER && !XMALLOC_OVERRIDE */
1901#endif
1902
1903    #include <linux/limits.h>
1904
1905    #ifndef INT32_MAX
1906        #define INT32_MAX INT_MAX
1907    #endif
1908    #ifndef UINT32_MAX
1909        #define UINT32_MAX UINT_MAX
1910    #endif
1911
1912    /* Linux headers define these using C expressions, but we need
1913     * them to be evaluable by the preprocessor, for use in sp_int.h.
1914     */
1915    #if BITS_PER_LONG == 64
1916        /* NOLINTBEGIN(bugprone-sizeof-expression) */
1917        static_assert(sizeof(ULONG_MAX) == 8,
1918                       "BITS_PER_LONG is 64, but ULONG_MAX is not.");
1919        /* NOLINTEND(bugprone-sizeof-expression) */
1920
1921        #undef UCHAR_MAX
1922        #define UCHAR_MAX 255
1923        #undef USHRT_MAX
1924        #define USHRT_MAX 65535
1925        #undef UINT_MAX
1926        #define UINT_MAX 4294967295U
1927        #undef ULONG_MAX
1928        #define ULONG_MAX 18446744073709551615UL
1929        #undef ULLONG_MAX
1930        #define ULLONG_MAX ULONG_MAX
1931        #undef INT_MAX
1932        #define INT_MAX 2147483647
1933        #undef LONG_MAX
1934        #define LONG_MAX 9223372036854775807L
1935        #undef LLONG_MAX
1936        #define LLONG_MAX LONG_MAX
1937
1938    #elif BITS_PER_LONG == 32
1939
1940        static_assert(sizeof(ULONG_MAX) == 4,
1941                       "BITS_PER_LONG is 32, but ULONG_MAX is not.");
1942
1943        #undef UCHAR_MAX
1944        #define UCHAR_MAX 255
1945        #undef USHRT_MAX
1946        #define USHRT_MAX 65535
1947        #undef UINT_MAX
1948        #define UINT_MAX 4294967295U
1949        #undef ULONG_MAX
1950        #define ULONG_MAX 4294967295UL
1951        #undef INT_MAX
1952        #define INT_MAX 2147483647
1953        #undef LONG_MAX
1954        #define LONG_MAX 2147483647L
1955
1956        #undef ULLONG_MAX
1957        #undef LLONG_MAX
1958        #if BITS_PER_LONG_LONG == 64
1959            #define ULLONG_MAX 18446744073709551615UL
1960            #define LLONG_MAX 9223372036854775807L
1961        #else
1962            #undef NO_64BIT
1963            #define NO_64BIT
1964            #define ULLONG_MAX ULONG_MAX
1965            #define LLONG_MAX LONG_MAX
1966        #endif
1967
1968#else
1969        #error unexpected BITS_PER_LONG value.
1970#endif
1971
1972/* WC_DUMP_BACKTRACE_NONDEBUG is intended to dump a backtrace only if it hasn't
1973 * already been dumped by the called function.
1974 */
1975#if defined(WOLFSSL_DEBUG_TRACE_ERROR_CODES) && defined(WOLFSSL_DEBUG_BACKTRACE_ERROR_CODES)
1976    #define WC_DUMP_BACKTRACE_NONDEBUG WC_DO_NOTHING
1977#else
1978    #define WC_DUMP_BACKTRACE_NONDEBUG dump_stack()
1979#endif
1980
1981#endif /* LINUXKM_WC_PORT_H */