luna - wolfssl/wolfcrypt/src/port/intel/quickassist

   1/* quickassist_sync.c
   2 *
   3 * Copyright (C) 2006-2026 wolfSSL Inc.
   4 *
   5 * This file is part of wolfSSL.
   6 *
   7 * wolfSSL is free software; you can redistribute it and/or modify
   8 * it under the terms of the GNU General Public License as published by
   9 * the Free Software Foundation; either version 3 of the License, or
  10 * (at your option) any later version.
  11 *
  12 * wolfSSL is distributed in the hope that it will be useful,
  13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15 * GNU General Public License for more details.
  16 *
  17 * You should have received a copy of the GNU General Public License
  18 * along with this program; if not, write to the Free Software
  19 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
  20 */
  21
  22#ifdef HAVE_CONFIG_H
  23    #include <config.h>
  24#endif
  25
  26#include <wolfssl/wolfcrypt/types.h>
  27
  28#ifdef HAVE_INTEL_QA_SYNC
  29
  30#ifdef QAT_DEMO_MAIN
  31    #define QAT_DEBUG
  32#endif
  33
  34
  35#include <wolfssl/internal.h>
  36#include <wolfssl/error-ssl.h>
  37#include <wolfssl/wolfcrypt/error-crypt.h>
  38#ifndef NO_AES
  39    #include <wolfssl/wolfcrypt/aes.h>
  40#endif
  41
  42#include <wolfssl/wolfcrypt/cryptocb.h>
  43#include <wolfssl/wolfcrypt/port/intel/quickassist_sync.h>
  44
  45#include "cpa.h"
  46#include "cpa_cy_im.h"
  47#include "cpa_cy_sym.h"
  48#include "cpa_cy_rsa.h"
  49#include "cpa_cy_ln.h"
  50#include "cpa_cy_ecdh.h"
  51#include "cpa_cy_ecdsa.h"
  52#include "cpa_cy_dh.h"
  53#include "cpa_cy_drbg.h"
  54#include "cpa_cy_nrbg.h"
  55#include "cpa_cy_prime.h"
  56
  57#include "icp_sal_user.h"
  58#include "icp_sal_poll.h"
  59
  60
  61#ifdef NO_INLINE
  62    #include <wolfssl/wolfcrypt/misc.h>
  63#else
  64    #define WOLFSSL_MISC_INCLUDED
  65    #include <wolfcrypt/src/misc.c>
  66#endif
  67
  68/* User space utils */
  69#include <stdio.h>
  70#include <stdlib.h>
  71#include <string.h>
  72#include <errno.h>
  73
  74#if 0
  75    /* Optional feature for partial QAT hashing support */
  76    /* This will process updates through hardware instead of caching them */
  77    #define QAT_HASH_ENABLE_PARTIAL
  78#endif
  79#ifdef QAT_HASH_ENABLE_PARTIAL
  80    #define MAX_QAT_HASH_BUFFERS 2
  81#endif
  82
  83/* Detect QAT driver version */
  84#if defined(CPA_CY_API_VERSION_NUM_MAJOR) && CPA_CY_API_VERSION_NUM_MAJOR > 1
  85    #define QAT_V2
  86#endif
  87
  88#ifdef QAT_V2
  89    /* quickassist/utilities/libusdm_drv/qae_mem.h */
  90    /* Provides user-space API's for accessing NUMA allocated memory through usdm_drv */
  91    #include "qae_mem.h"
  92#include "linux/include/qae_mem_utils.h"
  93#endif
  94
  95#ifdef QAT_USE_POLLING_THREAD
  96    #include <pthread.h>
  97#endif
  98
  99/* Tunable parameters */
 100#ifndef QAT_PROCESS_NAME
 101    #define QAT_PROCESS_NAME     "SSL"
 102#endif
 103#ifndef QAT_LIMIT_DEV_ACCESS
 104    #define QAT_LIMIT_DEV_ACCESS CPA_FALSE
 105#endif
 106#ifndef QAT_MAX_DEVICES
 107    #define QAT_MAX_DEVICES  (1)  /* maximum number of QAT cards */
 108#endif
 109
 110#ifndef QAT_RETRY_LIMIT
 111    #define QAT_RETRY_LIMIT  (100)
 112#endif
 113#ifndef QAT_POLL_RESP_QUOTA
 114    #define QAT_POLL_RESP_QUOTA (0) /* all pending */
 115#endif
 116
 117#if !defined(NO_AES) || !defined(NO_DES3)
 118    #define QAT_ENABLE_CRYPTO
 119#endif
 120
 121/* Pre-declarations */
 122struct IntelQaDev;
 123struct wc_CryptoInfo;
 124struct WC_BIGINT;
 125struct WC_RNG;
 126
 127
 128#if defined(QAT_ENABLE_HASH) || defined(QAT_ENABLE_CRYPTO)
 129/* symmetric context */
 130typedef struct IntelQaSymCtx {
 131    CpaCySymOpData opData;
 132    CpaCySymSessionCtx symCtxSrc;
 133    CpaCySymSessionCtx symCtx;
 134    word32 symCtxSize;
 135
 136    /* flags */
 137    word32 isOpen:1;
 138    word32 isCopy:1;
 139} IntelQaSymCtx;
 140#endif
 141
 142typedef void (*IntelQaFreeFunc)(struct IntelQaDev*);
 143
 144
 145/* QuickAssist device */
 146typedef struct IntelQaDev {
 147        CpaInstanceHandle handle;
 148    int devId;
 149        void* heap;
 150
 151    /* callback return info */
 152    int ret;
 153    byte* out;
 154    union {
 155        word32* outLenPtr;
 156        word32 outLen;
 157    };
 158
 159    /* operations */
 160    IntelQaFreeFunc freeFunc;
 161    union {
 162    #ifdef QAT_ENABLE_CRYPTO
 163        struct {
 164            IntelQaSymCtx ctx;
 165            CpaBufferList bufferList;
 166            CpaFlatBuffer flatBuffer;
 167            byte* authTag;
 168            word32 authTagSz;
 169        } cipher;
 170    #endif
 171    } op;
 172
 173#ifdef QAT_USE_POLLING_THREAD
 174    pthread_t pollingThread;
 175    byte pollingCy;
 176#endif
 177} IntelQaDev;
 178
 179
 180/* Interface */
 181static int IntelQaHardwareStart(const char*, int);
 182static void IntelQaHardwareStop(void);
 183static int IntelQaInit(void*);
 184static void IntelQaDeInit(int);
 185static int IntelQaNumInstances(void);
 186static int IntelQaOpen(IntelQaDev*, int);
 187static void IntelQaClose(IntelQaDev*);
 188static int IntelQaDevCopy(IntelQaDev*, IntelQaDev*);
 189static int IntelQaPoll(IntelQaDev*);
 190static int IntelQaGetCyInstanceCount(void);
 191
 192#ifndef NO_AES
 193    #ifdef HAVE_AES_CBC
 194        static int IntelQaSymAesCbcEncrypt(IntelQaDev*, byte*,
 195                const byte*, word32, const byte*, word32, const byte*, word32);
 196    #ifdef HAVE_AES_DECRYPT
 197        static int IntelQaSymAesCbcDecrypt(IntelQaDev*, byte*,
 198                const byte*, word32, const byte*, word32, const byte*, word32);
 199    #endif /* HAVE_AES_DECRYPT */
 200    #endif /* HAVE_AES_CBC */
 201
 202    #ifdef HAVE_AESGCM
 203        static int IntelQaSymAesGcmEncrypt(IntelQaDev*, byte*,
 204                const byte*, word32, const byte*, word32, const byte*, word32,
 205                byte*, word32, const byte*, word32);
 206    #ifdef HAVE_AES_DECRYPT
 207        static int IntelQaSymAesGcmDecrypt(IntelQaDev*, byte*,
 208                const byte*, word32, const byte*, word32, const byte*, word32,
 209                const byte*, word32, const byte*, word32);
 210    #endif /* HAVE_AES_DECRYPT */
 211    #endif /* HAVE_AESGCM */
 212#endif /* !NO_AES */
 213
 214#ifndef NO_DES3
 215    static int IntelQaSymDes3CbcEncrypt(IntelQaDev*, byte*,
 216            const byte*, word32, const byte*, word32, const byte* iv, word32);
 217    static int IntelQaSymDes3CbcDecrypt(IntelQaDev* dev, byte*,
 218            const byte*, word32, const byte*, word32, const byte* iv, word32);
 219#endif /*! NO_DES3 */
 220
 221#ifdef WOLF_CRYPTO_CB
 222    static int IntelQaSymSync_CryptoDevCb(int, struct wc_CryptoInfo*,
 223                        void*);
 224#endif /* WOLF_CRYPTO_CB */
 225
 226
 227#ifdef QAT_DEBUG
 228    #define QLOG(...) do { printf(__VA_ARGS__); } while (0)
 229#else
 230    #define QLOG(...) WC_DO_NOTHING
 231#endif
 232
 233
 234#define OS_HOST_TO_NW_32(uData) ByteReverseWord32(uData)
 235
 236
 237static CpaInstanceHandle* g_cyInstances = NULL;
 238static CpaInstanceInfo2* g_cyInstanceInfo = NULL;
 239static Cpa32U* g_cyInstMap = NULL;
 240static Cpa16U g_numInstances = 0;
 241static Cpa16U g_instCounter = 0;
 242static CpaBoolean g_cyServiceStarted = CPA_FALSE;
 243#ifdef QAT_USE_POLLING_CHECK
 244    static CpaBoolean* g_cyPolling = NULL;
 245    static pthread_mutex_t* g_PollLock;
 246#endif
 247static volatile int g_initCount = 0;
 248static pthread_mutex_t g_Hwlock = PTHREAD_MUTEX_INITIALIZER;
 249
 250
 251typedef struct qatCapabilities {
 252    /* capabilities */
 253    word32 supPartial:1;
 254    word32 supSha3:1;
 255} qatCapabilities_t;
 256static qatCapabilities_t g_qatCapabilities = {0};
 257
 258
 259#if defined(QAT_ENABLE_CRYPTO)
 260    static int IntelQaSymClose(IntelQaDev* dev, int doFree);
 261#endif
 262
 263
 264extern Cpa32U osalLogLevelSet(Cpa32U level);
 265
 266
 267static IntelQaDev qaDev;
 268
 269
 270/* -------------------------------------------------------------------------- */
 271/* Polling */
 272/* -------------------------------------------------------------------------- */
 273
 274static WC_INLINE int SyncSleep(word32 ms)
 275{
 276    int ret = 0;
 277    struct timespec resTime, remTime;
 278    resTime.tv_sec = ms/1000;
 279    resTime.tv_nsec = (ms%1000)*1000000;
 280    do {
 281        ret = nanosleep(&resTime, &remTime);
 282        resTime = remTime;
 283    } while ((ret!=0) && (errno == EINTR));
 284
 285    if (ret != 0) {
 286        QLOG("nanoSleep failed with code %d\n", ret);
 287        return BAD_FUNC_ARG;
 288    }
 289
 290   return ret;
 291}
 292
 293#ifdef QAT_USE_POLLING_THREAD
 294static void* IntelQaPollingThread(void* context)
 295{
 296    IntelQaDev* dev = (IntelQaDev*)context;
 297
 298    QLOG("Polling Thread Start\n");
 299    while (dev->pollingCy) {
 300        icp_sal_CyPollInstance(dev->handle, QAT_POLL_RESP_QUOTA);
 301        SyncSleep(10);
 302    }
 303    QLOG("Polling Thread Exit\n");
 304    pthread_exit(NULL);
 305}
 306
 307static CpaStatus IntelQaStartPollingThread(IntelQaDev* dev)
 308{
 309    if (dev->pollingCy == 0) {
 310        dev->pollingCy = 1;
 311
 312        QLOG("Polling Thread Created\n");
 313
 314        if (pthread_create(&dev->pollingThread, NULL, IntelQaPollingThread,
 315                                                            (void*)dev) != 0) {
 316            QLOG("Failed create polling thread!\n");
 317            return CPA_STATUS_FAIL;
 318        }
 319    }
 320    return CPA_STATUS_SUCCESS;
 321}
 322
 323static void IntelQaStopPollingThread(IntelQaDev* dev)
 324{
 325    dev->pollingCy = 0;
 326    pthread_join(dev->pollingThread, 0);
 327}
 328#endif /* QAT_USE_POLLING_THREAD */
 329
 330
 331/* -------------------------------------------------------------------------- */
 332/* Device */
 333/* -------------------------------------------------------------------------- */
 334void IntelQaHardwareStop(void)
 335{
 336    int i;
 337    CpaStatus status;
 338
 339    g_initCount--;  /* track de-init count */
 340    if (g_initCount != 0) {
 341        return;
 342    }
 343
 344    if (g_cyServiceStarted == CPA_TRUE) {
 345        g_cyServiceStarted = CPA_FALSE;
 346        for (i=0; i<g_numInstances; i++) {
 347            status = cpaCyStopInstance(g_cyInstances[i]);
 348            if (status != CPA_STATUS_SUCCESS) {
 349                QLOG("IntelQA: Could not stop instance: %d\n"
 350                        "\tInternal error has occur which probably can only be"
 351                        "fixed by a reboot\n", i);
 352            }
 353        }
 354    }
 355
 356    status = icp_sal_userStop();
 357    if (status != CPA_STATUS_SUCCESS) {
 358        QLOG("IntelQA: Could not stop sal for user space (status %d)\n",
 359                status);
 360    }
 361
 362    XFREE(g_cyInstMap, NULL, DYNAMIC_TYPE_ASYNC);
 363    g_cyInstMap = NULL;
 364
 365    XFREE(g_cyInstanceInfo, NULL, DYNAMIC_TYPE_ASYNC);
 366    g_cyInstanceInfo = NULL;
 367
 368#ifdef QAT_USE_POLLING_CHECK
 369    XFREE(g_cyPolling, NULL, DYNAMIC_TYPE_ASYNC);
 370    g_cyPolling = NULL;
 371    if (g_PollLock) {
 372        for (i=0; i<g_numInstances; i++) {
 373            pthread_mutex_destroy(&g_PollLock[i]);
 374        }
 375        XFREE(g_PollLock, NULL, DYNAMIC_TYPE_ASYNC);
 376        g_PollLock = NULL;
 377    }
 378#endif
 379
 380    if (g_cyInstances) {
 381        XFREE(g_cyInstances, NULL, DYNAMIC_TYPE_ASYNC);
 382        g_cyInstances = NULL;
 383        g_numInstances = 0;
 384    }
 385
 386    qaeMemDestroy();
 387
 388    QLOG("IntelQA: Stop\n");
 389}
 390
 391
 392int IntelQaHardwareStart(const char* process_name, int limitDevAccess)
 393{
 394    int ret = 0, i;
 395    CpaStatus status;
 396
 397    g_initCount++;
 398    if (g_initCount > 1) {
 399        return 0;
 400    }
 401
 402    status = qaeMemInit();
 403    if (status != CPA_STATUS_SUCCESS) {
 404        QLOG("IntelQA: Could not start qae mem for user space (status %d)\n"
 405                "\tHas the qaeMemDrv.ko module been loaded?\n",
 406                status);
 407        return ASYNC_INIT_E;
 408    }
 409
 410    status = icp_sal_userStartMultiProcess(process_name,
 411        limitDevAccess ? CPA_TRUE : CPA_FALSE);
 412    if (status != CPA_STATUS_SUCCESS) {
 413        QLOG("IntelQA: Could not start sal for user space! status %d\n",
 414                status);
 415        ret = ASYNC_INIT_E; goto error;
 416    }
 417
 418#ifdef QAT_DEBUG
 419    /* optionally enable debugging */
 420    /* osalLogLevelSet(8); */
 421#endif
 422
 423    status = cpaCyGetNumInstances(&g_numInstances);
 424    if (status != CPA_STATUS_SUCCESS || g_numInstances == 0) {
 425        QLOG("IntelQA: Failed to get num of instances! status %d\n", status);
 426        ret = INVALID_DEVID; goto error;
 427    }
 428
 429    /* Get handles / info */
 430    g_cyInstances = (CpaInstanceHandle*)XMALLOC(
 431        sizeof(CpaInstanceHandle) * g_numInstances, NULL, DYNAMIC_TYPE_ASYNC);
 432    if (g_cyInstances == NULL) {
 433        QLOG("IntelQA: Failed to allocate instances\n");
 434        ret = INVALID_DEVID; goto error;
 435    }
 436
 437#ifdef QAT_USE_POLLING_CHECK
 438    g_cyPolling = (CpaBoolean*)XMALLOC(sizeof(CpaBoolean) * g_numInstances, NULL,
 439        DYNAMIC_TYPE_ASYNC);
 440    if (g_cyPolling == NULL) {
 441        QLOG("IntelQA: Failed to allocate polling status\n");
 442        ret = INVALID_DEVID; goto error;
 443    }
 444    g_PollLock = (pthread_mutex_t*)XMALLOC(sizeof(pthread_mutex_t) *
 445        g_numInstances, NULL, DYNAMIC_TYPE_ASYNC);
 446    if (g_PollLock == NULL) {
 447        QLOG("IntelQA: Failed to allocate polling locks\n");
 448        ret = INVALID_DEVID; goto error;
 449    }
 450    for (i=0; i<g_numInstances; i++) {
 451        pthread_mutex_init(&g_PollLock[i], NULL);
 452    }
 453#endif
 454
 455    g_cyInstanceInfo = (CpaInstanceInfo2*)XMALLOC(
 456        sizeof(CpaInstanceInfo2) * g_numInstances, NULL, DYNAMIC_TYPE_ASYNC);
 457    if (g_cyInstanceInfo == NULL) {
 458        QLOG("IntelQA: Failed to allocate instance info\n");
 459        ret = INVALID_DEVID; goto error;
 460    }
 461
 462    g_cyInstMap = (Cpa32U*)XMALLOC(
 463        sizeof(Cpa32U) * g_numInstances, NULL, DYNAMIC_TYPE_ASYNC);
 464    if (g_cyInstMap == NULL) {
 465        QLOG("IntelQA: Failed to allocate instance map\n");
 466        ret = INVALID_DEVID; goto error;
 467    }
 468
 469    status = cpaCyGetInstances(g_numInstances, g_cyInstances);
 470    if (status != CPA_STATUS_SUCCESS) {
 471        QLOG("IntelQA: Failed to get IntelQA instances\n");
 472        ret = INVALID_DEVID; goto error;
 473    }
 474
 475    /* start all instances */
 476    g_cyServiceStarted = CPA_TRUE;
 477    for (i=0; i<g_numInstances; i++) {
 478        Cpa32U coreAffinity = 0;
 479        CpaCySymCapabilitiesInfo capabilities;
 480        int j;
 481        XMEMSET(&capabilities, 0, sizeof(capabilities));
 482
 483        status = cpaCyInstanceGetInfo2(g_cyInstances[i],
 484                                                    &g_cyInstanceInfo[i]);
 485        if (status != CPA_STATUS_SUCCESS) {
 486            QLOG("IntelQA: Error getting instance info for %d\n", i);
 487            ret = INVALID_DEVID; goto error;
 488        }
 489
 490        /* loop of the instanceInfo coreAffinity bitmask to find the core */
 491        for (j=0; j<CPA_MAX_CORES; j++) {
 492            if (CPA_BITMAP_BIT_TEST(g_cyInstanceInfo[i].coreAffinity, j)) {
 493                coreAffinity = i;
 494                break;
 495            }
 496        }
 497        g_cyInstMap[i] = coreAffinity;
 498
 499        /* capabilities */
 500        status = cpaCySymQueryCapabilities(g_cyInstances[i], &capabilities);
 501        if (status == CPA_STATUS_SUCCESS) {
 502            g_qatCapabilities.supPartial = capabilities.partialPacketSupported;
 503            if (capabilities.partialPacketSupported != CPA_TRUE) {
 504                QLOG("Warning: QAT does not support partial packets!\n");
 505            }
 506        }
 507
 508        QLOG("Inst %d, Node: %d, Affin: %u, Dev: %u, Accel %u, "
 509                "EE %u, BDF %02X:%02X:%02X, isPolled %d\n",
 510                i, g_cyInstanceInfo[i].nodeAffinity, coreAffinity,
 511                g_cyInstanceInfo[i].physInstId.packageId,
 512                g_cyInstanceInfo[i].physInstId.acceleratorId,
 513                g_cyInstanceInfo[i].physInstId.executionEngineId,
 514                (Cpa8U)((g_cyInstanceInfo[i].physInstId.busAddress) >> 8),
 515                (Cpa8U)((g_cyInstanceInfo[i].physInstId.busAddress)
 516                                                            & 0xFF) >> 3,
 517                (Cpa8U)((g_cyInstanceInfo[i].physInstId.busAddress) & 3),
 518                g_cyInstanceInfo[i].isPolled);
 519
 520        status = cpaCySetAddressTranslation(g_cyInstances[i],
 521            qaeVirtToPhysNUMA);
 522        if (status != CPA_STATUS_SUCCESS) {
 523            QLOG("IntelQA: Error setting memory config for inst %d\n", i);
 524            ret = INVALID_DEVID; goto error;
 525        }
 526
 527        status = cpaCyStartInstance(g_cyInstances[i]);
 528        if (status != CPA_STATUS_SUCCESS) {
 529            QLOG("IntelQA: Error starting crypto instance %d\n", i);
 530            ret = INVALID_DEVID; goto error;
 531        }
 532    }
 533
 534    QLOG("IntelQA: Instances %d\n", g_numInstances);
 535    return ret;
 536
 537error:
 538    IntelQaHardwareStop();
 539    return ret;
 540}
 541
 542
 543int IntelQaInit(void* threadId)
 544{
 545    int ret;
 546    int devId;
 547    (void)threadId;
 548
 549    ret = pthread_mutex_lock(&g_Hwlock);
 550    if (ret != 0) {
 551        QLOG("IntelQaInit: mutex lock failed! %d\n", ret);
 552        return BAD_MUTEX_E;
 553    }
 554
 555    ret = IntelQaHardwareStart(QAT_PROCESS_NAME, QAT_LIMIT_DEV_ACCESS);
 556    if (ret != 0) {
 557        pthread_mutex_unlock(&g_Hwlock);
 558        return ret;
 559    }
 560
 561    if (g_numInstances <= 0) {
 562        pthread_mutex_unlock(&g_Hwlock);
 563        return ASYNC_INIT_E;
 564    }
 565
 566    /* assign device id */
 567    devId = (g_instCounter % g_numInstances);
 568    g_instCounter++;
 569
 570    pthread_mutex_unlock(&g_Hwlock);
 571
 572    return devId;
 573}
 574
 575
 576int IntelQaNumInstances(void)
 577{
 578    return g_numInstances;
 579}
 580
 581
 582int IntelQaOpen(IntelQaDev* dev, int devId)
 583{
 584    if (dev == NULL) {
 585        return BAD_FUNC_ARG;
 586    }
 587
 588    /* clear device info */
 589    XMEMSET(dev, 0, sizeof(IntelQaDev));
 590
 591    if (g_cyInstances == NULL) {
 592        QLOG("IntelQA not initialized\n");
 593        return ASYNC_INIT_E;
 594    }
 595
 596    dev->devId = devId;
 597    dev->handle = g_cyInstances[devId];
 598
 599#ifdef QAT_USE_POLLING_THREAD
 600    /* start polling thread */
 601    IntelQaStartPollingThread(dev);
 602#endif
 603
 604    return 0;
 605}
 606
 607
 608#if defined(QAT_ENABLE_CRYPTO)
 609
 610static IntelQaSymCtx* IntelQaGetSymCtx(IntelQaDev* dev)
 611{
 612    return &dev->op.cipher.ctx;
 613}
 614
 615#endif
 616
 617
 618void IntelQaClose(IntelQaDev* dev)
 619{
 620    if (dev) {
 621        QLOG("IntelQaClose %p\n", dev);
 622        /* close any active session */
 623        IntelQaSymClose(dev, 1);
 624
 625    #ifdef QAT_USE_POLLING_THREAD
 626        IntelQaStopPollingThread(dev);
 627    #endif
 628
 629        dev->handle = NULL;
 630    }
 631}
 632
 633void IntelQaDeInit(int devId)
 634{
 635    (void)devId;
 636
 637    if (pthread_mutex_lock(&g_Hwlock) == 0) {
 638        IntelQaHardwareStop();
 639        pthread_mutex_unlock(&g_Hwlock);
 640    }
 641}
 642
 643int IntelQaPoll(IntelQaDev* dev)
 644{
 645    int ret = 0;
 646    CpaStatus status;
 647
 648#ifdef QAT_USE_POLLING_CHECK
 649    pthread_mutex_t* lock = &g_PollLock[dev->qat.devId];
 650    if (pthread_mutex_lock(lock) == 0) {
 651        /* test if any other threads are polling */
 652        if (g_cyPolling[dev->qat.devId]) {
 653            pthread_mutex_unlock(lock);
 654
 655            /* return success even though its busy, caller will treat as WC_PENDING_E */
 656            return 0;
 657        }
 658
 659        g_cyPolling[dev->qat.devId] = 1;
 660        pthread_mutex_unlock(lock);
 661    }
 662#endif
 663
 664    status = icp_sal_CyPollInstance(dev->handle, QAT_POLL_RESP_QUOTA);
 665    if (status != CPA_STATUS_SUCCESS && status != CPA_STATUS_RETRY) {
 666        QLOG("IntelQa: Poll failure %d\n", status);
 667        ret = -1;
 668    }
 669
 670    {
 671        if (dev->ret != WC_NO_ERR_TRACE(WC_PENDING_E)) {
 672            /* perform cleanup */
 673            IntelQaFreeFunc freeFunc = dev->freeFunc;
 674            QLOG("IntelQaOpFree: Dev %p, FreeFunc %p\n", dev, freeFunc);
 675            if (freeFunc) {
 676                dev->freeFunc = NULL;
 677                freeFunc(dev);
 678            }
 679        }
 680    }
 681
 682#ifdef QAT_USE_POLLING_CHECK
 683    /* indicate we are done polling */
 684    if (pthread_mutex_lock(lock) == 0) {
 685        g_cyPolling[dev->qat.devId] = 0;
 686        pthread_mutex_unlock(lock);
 687    }
 688#endif
 689
 690    return ret;
 691}
 692
 693static int IntelQaPollBlockRet(IntelQaDev* dev, int ret_wait)
 694{
 695    int ret;
 696
 697    do {
 698        ret = IntelQaPoll(dev);
 699
 700        if (dev->ret != ret_wait) {
 701            break;
 702        }
 703    } while (1);
 704    ret = dev->ret;
 705
 706    return ret;
 707}
 708
 709int IntelQaGetCyInstanceCount(void)
 710{
 711    return g_numInstances;
 712}
 713
 714static WC_INLINE int IntelQaHandleCpaStatus(IntelQaDev* dev, CpaStatus status,
 715    int* ret, byte isAsync, void* callback, int* retryCount)
 716{
 717    int retry = 0;
 718
 719    if (status == CPA_STATUS_SUCCESS) {
 720        if (isAsync && callback) {
 721            *ret = WC_PENDING_E;
 722        }
 723        else {
 724            *ret = IntelQaPollBlockRet(dev, WC_PENDING_E);
 725        }
 726    }
 727    else if (status == CPA_STATUS_RETRY) {
 728        (*retryCount)++;
 729        if ((*retryCount % (QAT_RETRY_LIMIT + 1)) == QAT_RETRY_LIMIT) {
 730            SyncSleep(10);
 731        }
 732        retry = 1;
 733    }
 734    else {
 735        *ret = ASYNC_OP_E;
 736    }
 737
 738    return retry;
 739}
 740
 741static WC_INLINE void IntelQaOpInit(IntelQaDev* dev, IntelQaFreeFunc freeFunc)
 742{
 743    dev->ret = WC_PENDING_E;
 744    dev->freeFunc = freeFunc;
 745}
 746
 747
 748/* -------------------------------------------------------------------------- */
 749/* Symmetric Algos */
 750/* -------------------------------------------------------------------------- */
 751
 752#if defined(QAT_ENABLE_CRYPTO)
 753
 754static int IntelQaSymOpen(IntelQaDev* dev, CpaCySymSessionSetupData* setup,
 755    CpaCySymCbFunc callback)
 756{
 757    int ret = 0;
 758    CpaStatus status = CPA_STATUS_SUCCESS;
 759    Cpa32U sessionCtxSize = 0;
 760    IntelQaSymCtx* ctx;
 761
 762    /* arg check */
 763    if (dev == NULL || setup == NULL) {
 764        return BAD_FUNC_ARG;
 765    }
 766
 767    ctx = IntelQaGetSymCtx(dev);
 768
 769    /* Determine size of session context to allocate - use max size */
 770    status = cpaCySymSessionCtxGetSize(dev->handle, setup, &sessionCtxSize);
 771
 772    if (ctx->symCtxSize > 0 && ctx->symCtxSize > sessionCtxSize) {
 773        QLOG("Symmetric context size error! Buf %d, Exp %d\n",
 774                ctx->symCtxSize, sessionCtxSize);
 775        return ASYNC_OP_E;
 776    }
 777
 778    /* make sure session context is allocated */
 779    if (ctx->symCtx == NULL) {
 780        /* Allocate session context */
 781        ctx->symCtx = XMALLOC(sessionCtxSize, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA64);
 782        if (ctx->symCtx == NULL) {
 783            return MEMORY_E;
 784        }
 785    }
 786    ctx->symCtxSize = sessionCtxSize;
 787
 788    if (!ctx->isOpen) {
 789        ctx->isOpen = 1;
 790
 791        QLOG("IntelQaSymOpen: InitSession dev %p, symCtx %p\n",
 792                dev, ctx->symCtx);
 793
 794        /* open symmetric session */
 795        status = cpaCySymInitSession(dev->handle, callback, setup, ctx->symCtx);
 796        if (status != CPA_STATUS_SUCCESS) {
 797            QLOG("cpaCySymInitSession failed! dev %p, status %d\n",
 798                    dev, status);
 799            XFREE(ctx->symCtx, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA64);
 800            ctx->symCtx = NULL;
 801            return ASYNC_INIT_E;
 802        }
 803    }
 804
 805    if (ctx->symCtxSrc == NULL) {
 806        ctx->symCtxSrc = ctx->symCtx;
 807    }
 808
 809    QLOG("IntelQaSymOpen: dev %p, symCtx %p (src %p), "
 810            "symCtxSize %d, isCopy %d, isOpen %d\n",
 811            dev, ctx->symCtx, ctx->symCtxSrc, ctx->symCtxSize,
 812            ctx->isCopy, ctx->isOpen);
 813
 814    return ret;
 815}
 816
 817static int IntelQaSymClose(IntelQaDev* dev, int doFree)
 818{
 819    int ret = 0;
 820    CpaStatus status = CPA_STATUS_SUCCESS;
 821    IntelQaSymCtx* ctx;
 822
 823    if (dev == NULL) {
 824        return BAD_FUNC_ARG;
 825    }
 826
 827    ctx = IntelQaGetSymCtx(dev);
 828
 829    QLOG("IntelQaSymClose: dev %p, ctx %p, symCtx %p (src %p), "
 830            "symCtxSize %d, isCopy %d, isOpen %d, doFree %d\n",
 831            dev, ctx, ctx->symCtx, ctx->symCtxSrc, ctx->symCtxSize,
 832            ctx->isCopy, ctx->isOpen, doFree);
 833
 834    if (ctx->symCtx == ctx->symCtxSrc && ctx->symCtx != NULL) {
 835        if (ctx->isOpen) {
 836            ctx->isOpen = 0;
 837            QLOG("IntelQaSymClose: RemoveSession dev %p, symCtx %p\n",
 838                    dev, ctx->symCtx);
 839            status = cpaCySymRemoveSession(dev->handle, ctx->symCtx);
 840            if (status == CPA_STATUS_RETRY) {
 841                QLOG("cpaCySymRemoveSession retry!\n");
 842                /* treat this as error, since session should not be active */
 843                ret = ASYNC_OP_E;
 844            }
 845            else if (status != CPA_STATUS_SUCCESS) {
 846                QLOG("cpaCySymRemoveSession failed! status %d\n", status);
 847                ret = ASYNC_OP_E;
 848            }
 849        }
 850    }
 851
 852    if (doFree) {
 853        XFREE(ctx->symCtx, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA64);
 854        ctx->symCtx = NULL;
 855        ctx->symCtxSrc = NULL;
 856        ctx->symCtxSize = 0;
 857    }
 858
 859    return ret;
 860}
 861
 862#endif /* QAT_ENABLE_CRYPTO */
 863
 864
 865/* -------------------------------------------------------------------------- */
 866/* AES/DES Algo */
 867/* -------------------------------------------------------------------------- */
 868
 869#ifdef QAT_ENABLE_CRYPTO
 870
 871static void IntelQaSymCipherFree(IntelQaDev* dev)
 872{
 873    IntelQaSymCtx* ctx = &dev->op.cipher.ctx;
 874    CpaCySymOpData* opData = &ctx->opData;
 875    CpaBufferList* pDstBuffer = &dev->op.cipher.bufferList;
 876
 877    if (opData) {
 878        XFREE(opData->pAdditionalAuthData, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA);
 879        opData->pAdditionalAuthData = NULL;
 880        XFREE(opData->pIv, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA);
 881        opData->pIv = NULL;
 882        XMEMSET(opData, 0, sizeof(CpaCySymOpData));
 883    }
 884    if (pDstBuffer) {
 885        if (pDstBuffer->pBuffers) {
 886            XFREE(pDstBuffer->pBuffers->pData, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA);
 887            pDstBuffer->pBuffers->pData = NULL;
 888            XMEMSET(pDstBuffer->pBuffers, 0, sizeof(CpaFlatBuffer));
 889        }
 890        XFREE(pDstBuffer->pPrivateMetaData, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA);
 891        pDstBuffer->pPrivateMetaData = NULL;
 892        XMEMSET(pDstBuffer, 0, sizeof(CpaBufferList));
 893    }
 894
 895    /* close and free sym context */
 896    IntelQaSymClose(dev, 1);
 897
 898    /* clear temp pointers */
 899    dev->out = NULL;
 900    dev->outLen = 0;
 901#ifndef NO_AES
 902    if (dev->op.cipher.authTag != NULL) {
 903        XMEMSET(dev->op.cipher.authTag, 0, dev->op.cipher.authTagSz);
 904        XFREE(dev->op.cipher.authTag, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA);
 905        dev->op.cipher.authTag = NULL;
 906    }
 907    dev->op.cipher.authTagSz = 0;
 908#endif
 909}
 910
 911static int IntelQaSymCipher(IntelQaDev* dev, byte* out, const byte* in,
 912    word32 inOutSz, const byte* key, word32 keySz, const byte* iv, word32 ivSz,
 913    CpaCySymOp symOperation, CpaCySymCipherAlgorithm cipherAlgorithm,
 914    CpaCySymCipherDirection cipherDirection,
 915
 916    /* for auth ciphers (CCM or GCM) */
 917    CpaCySymHashAlgorithm hashAlgorithm,
 918    byte* authTag, word32 authTagSz,
 919    const byte* authIn, word32 authInSz)
 920{
 921    int ret;
 922    CpaStatus status = CPA_STATUS_SUCCESS;
 923    CpaCySymOpData* opData = NULL;
 924    CpaCySymSessionSetupData setup;
 925    const Cpa32U numBuffers = 1;
 926    CpaBufferList* bufferList = NULL;
 927    CpaFlatBuffer* flatBuffer = NULL;
 928    Cpa8U* ivBuf = NULL;
 929    Cpa8U* dataBuf = NULL;
 930    Cpa32U dataLen = inOutSz;
 931    Cpa8U* metaBuf = NULL;
 932    Cpa32U metaSize = 0;
 933    Cpa8U* authInBuf = NULL;
 934    Cpa32U authInSzAligned = authInSz;
 935    Cpa8U* authTagBuf = NULL;
 936    IntelQaSymCtx* ctx;
 937    CpaBoolean verifyResult = CPA_FALSE;
 938
 939    QLOG("IntelQaSymCipher: dev %p, out %p, in %p, inOutSz %d, op %d, "
 940            "algo %d, dir %d, hash %d\n",
 941            dev, out, in, inOutSz, symOperation, cipherAlgorithm,
 942            cipherDirection, hashAlgorithm);
 943
 944    /* check args */
 945    if (out == NULL || in == NULL || inOutSz == 0 ||
 946        key == NULL || keySz == 0 || iv == NULL || ivSz == 0) {
 947        return BAD_FUNC_ARG;
 948    }
 949    if (hashAlgorithm != CPA_CY_SYM_HASH_NONE &&
 950        (authTag == NULL || authTagSz == 0)) {
 951        return BAD_FUNC_ARG;
 952    }
 953
 954    /* get meta size */
 955    status = cpaCyBufferListGetMetaSize(dev->handle, numBuffers, &metaSize);
 956    if (status != CPA_STATUS_SUCCESS && metaSize <= 0) {
 957        ret = BUFFER_E; goto exit;
 958    }
 959
 960    /* if authtag provided then it will be appended to end of input */
 961    if (authTag && authTagSz > 0) {
 962        dataLen += authTagSz;
 963    }
 964
 965    /* allocate buffers */
 966    ctx = &dev->op.cipher.ctx;
 967    opData = &ctx->opData;
 968    bufferList = &dev->op.cipher.bufferList;
 969    flatBuffer = &dev->op.cipher.flatBuffer;
 970    metaBuf = XMALLOC(metaSize, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA);
 971    dataBuf = XMALLOC(dataLen, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA);
 972    XMEMCPY(dataBuf, in, inOutSz);
 973    ivBuf = XMALLOC(WC_AES_BLOCK_SIZE, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA);
 974    XMEMCPY(ivBuf, iv, ivSz);
 975    authTagBuf = XMALLOC(authTagSz, dev->heap, DYNAMIC_TYPE_ASYNC_NUMA);
 976
 977    /* check allocations */
 978    if (ivBuf == NULL || metaBuf == NULL || dataBuf == NULL ||
 979        authTagBuf == NULL) {
 980        ret = MEMORY_E; goto exit;
 981    }
 982
 983    /* AAD */
 984    if (authIn && authInSz > 0) {
 985        /* make sure AAD is block aligned */
 986        if (authInSzAligned % WC_AES_BLOCK_SIZE) {
 987            authInSzAligned += WC_AES_BLOCK_SIZE -
 988                (authInSzAligned % WC_AES_BLOCK_SIZE);
 989        }
 990
 991        authInBuf = XMALLOC(authInSzAligned, dev->heap,
 992                DYNAMIC_TYPE_ASYNC_NUMA);
 993        XMEMCPY(authInBuf, authIn, authInSz);
 994        if (authInBuf == NULL) {
 995            ret = MEMORY_E; goto exit;
 996        }
 997        /* clear remainder */
 998        XMEMSET(authInBuf + authInSz, 0, authInSzAligned - authInSz);
 999    }
1000
1001    /* init buffers */
1002    XMEMSET(&setup, 0, sizeof(CpaCySymSessionSetupData));
1003    XMEMSET(opData, 0, sizeof(CpaCySymOpData));
1004    XMEMSET(bufferList, 0, sizeof(CpaBufferList));
1005    XMEMSET(flatBuffer, 0, sizeof(CpaFlatBuffer));
1006    XMEMSET(metaBuf, 0, metaSize);
1007
1008    bufferList->pBuffers = flatBuffer;
1009    bufferList->numBuffers = numBuffers;
1010    bufferList->pPrivateMetaData = metaBuf;
1011    flatBuffer->dataLenInBytes = dataLen;
1012    flatBuffer->pData = dataBuf;
1013
1014    /* setup */
1015    setup.sessionPriority = CPA_CY_PRIORITY_NORMAL;
1016    setup.symOperation = symOperation;
1017    setup.cipherSetupData.cipherAlgorithm = cipherAlgorithm;
1018    setup.cipherSetupData.cipherKeyLenInBytes = keySz;
1019    setup.cipherSetupData.pCipherKey = (byte*)key;
1020    setup.cipherSetupData.cipherDirection = cipherDirection;
1021
1022    /* setup auth ciphers */
1023    if (hashAlgorithm != CPA_CY_SYM_HASH_NONE) {
1024        setup.algChainOrder =
1025            (cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT) ?
1026                CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH :
1027                CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
1028
1029        setup.hashSetupData.hashAlgorithm = hashAlgorithm;
1030        setup.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_AUTH;
1031        setup.hashSetupData.digestResultLenInBytes = authTagSz;
1032        setup.hashSetupData.authModeSetupData.aadLenInBytes = authInSz;
1033
1034        if (cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT)
1035            setup.digestIsAppended = CPA_TRUE;
1036        else
1037            setup.digestIsAppended = CPA_FALSE;
1038    }
1039
1040    /* open session */
1041    ret = IntelQaSymOpen(dev, &setup, NULL);
1042    if (ret != 0) {
1043        goto exit;
1044    }
1045
1046    /* operation data */
1047    opData->sessionCtx = ctx->symCtx;
1048    opData->packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
1049    opData->pIv = ivBuf;
1050    opData->ivLenInBytes = ivSz;
1051    opData->cryptoStartSrcOffsetInBytes = 0;
1052    opData->messageLenToCipherInBytes = inOutSz;
1053    if (authIn && authInSz > 0) {
1054        opData->pAdditionalAuthData = authInBuf;
1055    }
1056    if (cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT) {
1057        if (authTag && authTagSz > 0) {
1058            /* append digest to end of data buffer */
1059            XMEMCPY(flatBuffer->pData + inOutSz, authTag, authTagSz);
1060        }
1061    }
1062    else {
1063        if (authTag && authTagSz > 0) {
1064            XMEMCPY(authTagBuf, authTag, authTagSz);
1065        }
1066    }
1067
1068    /* store info needed for output */
1069    dev->out = out;
1070    dev->outLen = inOutSz;
1071    if (cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT) {
1072        dev->op.cipher.authTag = authTagBuf;
1073        dev->op.cipher.authTagSz = authTagSz;
1074        opData->pDigestResult = authTagBuf;
1075    }
1076    else {
1077        dev->op.cipher.authTag = NULL;
1078        dev->op.cipher.authTagSz = 0;
1079    }
1080    IntelQaOpInit(dev, IntelQaSymCipherFree);
1081
1082    /* perform symmetric AES operation async */
1083    /* use same buffer list for in-place operation */
1084    status = cpaCySymPerformOp(dev->handle, dev, opData,
1085            bufferList, bufferList, &verifyResult);
1086
1087    if (symOperation == CPA_CY_SYM_OP_ALGORITHM_CHAINING &&
1088        cipherAlgorithm == CPA_CY_SYM_CIPHER_AES_GCM &&
1089        cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT &&
1090        hashAlgorithm == CPA_CY_SYM_HASH_AES_GCM) {
1091        if (verifyResult == CPA_FALSE) {
1092            ret = AES_GCM_AUTH_E;
1093        }
1094    }
1095exit:
1096
1097    if (ret != 0) {
1098        QLOG("cpaCySymPerformOp Cipher failed! dev %p, status %d, ret %d\n",
1099            dev, status, ret);
1100    }
1101
1102    /* Capture the inline decrypt into the output. */
1103    XMEMCPY(out, dataBuf, inOutSz);
1104    if (cipherDirection == CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT) {
1105        if (authTag != NULL && authTagSz > 0) {
1106            XMEMCPY(authTag, authTagBuf, authTagSz);
1107        }
1108    }
1109
1110    /* handle cleanup */
1111    IntelQaSymCipherFree(dev);
1112
1113    return ret;
1114}
1115
1116#ifdef HAVE_AES_CBC
1117int IntelQaSymAesCbcEncrypt(IntelQaDev* dev,
1118            byte* out, const byte* in, word32 sz,
1119            const byte* key, word32 keySz,
1120            const byte* iv, word32 ivSz)
1121{
1122    int ret = IntelQaSymCipher(dev, out, in, sz,
1123        key, keySz, iv, ivSz,
1124        CPA_CY_SYM_OP_CIPHER, CPA_CY_SYM_CIPHER_AES_CBC,
1125        CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT,
1126        CPA_CY_SYM_HASH_NONE, NULL, 0, NULL, 0);
1127
1128    XMEMCPY((byte*)iv, out + sz - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
1129    return ret;
1130}
1131
1132#ifdef HAVE_AES_DECRYPT
1133int IntelQaSymAesCbcDecrypt(IntelQaDev* dev,
1134            byte* out, const byte* in, word32 sz,
1135            const byte* key, word32 keySz,
1136            const byte* iv, word32 ivSz)
1137{
1138    byte nextIv[WC_AES_BLOCK_SIZE];
1139    int ret;
1140
1141    XMEMCPY(nextIv, in + sz - WC_AES_BLOCK_SIZE, WC_AES_BLOCK_SIZE);
1142    ret = IntelQaSymCipher(dev, out, in, sz,
1143        key, keySz, iv, ivSz,
1144        CPA_CY_SYM_OP_CIPHER, CPA_CY_SYM_CIPHER_AES_CBC,
1145        CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT,
1146        CPA_CY_SYM_HASH_NONE, NULL, 0, NULL, 0);
1147
1148    XMEMCPY((byte*)iv, nextIv, WC_AES_BLOCK_SIZE);
1149    return ret;
1150}
1151#endif /* HAVE_AES_DECRYPT */
1152#endif /* HAVE_AES_CBC */
1153
1154
1155#ifdef HAVE_AESGCM
1156int IntelQaSymAesGcmEncrypt(IntelQaDev* dev,
1157            byte* out, const byte* in, word32 sz,
1158            const byte* key, word32 keySz,
1159            const byte* iv, word32 ivSz,
1160            byte* authTag, word32 authTagSz,
1161            const byte* authIn, word32 authInSz)
1162{
1163    return IntelQaSymCipher(dev, out, in, sz,
1164        key, keySz, iv, ivSz,
1165        CPA_CY_SYM_OP_ALGORITHM_CHAINING, CPA_CY_SYM_CIPHER_AES_GCM,
1166        CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT,
1167        CPA_CY_SYM_HASH_AES_GCM, authTag, authTagSz, authIn, authInSz);
1168}
1169#ifdef HAVE_AES_DECRYPT
1170int IntelQaSymAesGcmDecrypt(IntelQaDev* dev,
1171            byte* out, const byte* in, word32 sz,
1172            const byte* key, word32 keySz,
1173            const byte* iv, word32 ivSz,
1174            const byte* authTag, word32 authTagSz,
1175            const byte* authIn, word32 authInSz)
1176{
1177    return IntelQaSymCipher(dev, out, in, sz,
1178        key, keySz, iv, ivSz,
1179        CPA_CY_SYM_OP_ALGORITHM_CHAINING, CPA_CY_SYM_CIPHER_AES_GCM,
1180        CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT,
1181        CPA_CY_SYM_HASH_AES_GCM, (byte*)authTag, authTagSz, authIn, authInSz);
1182}
1183#endif /* HAVE_AES_DECRYPT */
1184#endif /* HAVE_AESGCM */
1185
1186#ifndef NO_DES3
1187int IntelQaSymDes3CbcEncrypt(IntelQaDev* dev,
1188            byte* out, const byte* in, word32 sz,
1189            const byte* key, word32 keySz,
1190            const byte* iv, word32 ivSz)
1191{
1192    return IntelQaSymCipher(dev, out, in, sz,
1193        key, keySz, iv, ivSz,
1194        CPA_CY_SYM_OP_CIPHER, CPA_CY_SYM_CIPHER_3DES_CBC,
1195        CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT,
1196        CPA_CY_SYM_HASH_NONE, NULL, 0, NULL, 0);
1197}
1198
1199int IntelQaSymDes3CbcDecrypt(IntelQaDev* dev,
1200            byte* out, const byte* in, word32 sz,
1201            const byte* key, word32 keySz,
1202            const byte* iv, word32 ivSz)
1203{
1204    return IntelQaSymCipher(dev, out, in, sz,
1205        key, keySz, iv, ivSz,
1206        CPA_CY_SYM_OP_CIPHER, CPA_CY_SYM_CIPHER_3DES_CBC,
1207        CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT,
1208        CPA_CY_SYM_HASH_NONE, NULL, 0, NULL, 0);
1209}
1210#endif /* !NO_DES3 */
1211
1212#endif /* QAT_ENABLE_CRYPTO */
1213
1214
1215#ifdef WOLF_CRYPTO_CB
1216
1217int IntelQaSymSync_CryptoDevCb(int devId, struct wc_CryptoInfo* info, void* ctx)
1218{
1219    int rc = NOT_COMPILED_IN; /* return this to bypass HW and use SW */
1220    IntelQaDev* dev;
1221
1222    if (info == NULL || ctx == NULL)
1223        return BAD_FUNC_ARG;
1224
1225    (void)devId;
1226    dev = (IntelQaDev*)ctx;
1227
1228    #ifdef QAT_ENABLE_CRYPTO
1229    if (info->algo_type == WC_ALGO_TYPE_CIPHER) {
1230        QLOG("CryptoDevCb Cipher: Type %d\n", info->cipher.type);
1231
1232        #ifndef NO_AES
1233        if (info->cipher.type == WC_CIPHER_AES_CBC) {
1234            Aes* aes = info->cipher.aescbc.aes;
1235            if (aes == NULL)
1236                return BAD_FUNC_ARG;
1237
1238            if (info->cipher.enc) {
1239                rc = IntelQaSymAesCbcEncrypt(dev,
1240                        info->cipher.aescbc.out,
1241                        info->cipher.aescbc.in,
1242                        info->cipher.aescbc.sz,
1243                        (byte*)aes->devKey, aes->keylen,
1244                        (byte*)aes->reg, WC_AES_BLOCK_SIZE);
1245            }
1246            else {
1247                rc = IntelQaSymAesCbcDecrypt(dev,
1248                        info->cipher.aescbc.out,
1249                        info->cipher.aescbc.in,
1250                        info->cipher.aescbc.sz,
1251                        (byte*)aes->devKey, aes->keylen,
1252                        (byte*)aes->reg, WC_AES_BLOCK_SIZE);
1253            }
1254        }
1255        #endif /* !NO_AES */
1256
1257        #ifdef HAVE_AESGCM
1258        if (info->cipher.type == WC_CIPHER_AES_GCM) {
1259            if (info->cipher.enc) {
1260                Aes* aes = info->cipher.aesgcm_enc.aes;
1261                if (aes == NULL)
1262                    return BAD_FUNC_ARG;
1263
1264                rc = IntelQaSymAesGcmEncrypt(dev,
1265                        info->cipher.aesgcm_enc.out,
1266                        info->cipher.aesgcm_enc.in,
1267                        info->cipher.aesgcm_enc.sz,
1268                        (const byte*)aes->devKey, aes->keylen,
1269                        info->cipher.aesgcm_enc.iv,
1270                        info->cipher.aesgcm_enc.ivSz,
1271                        info->cipher.aesgcm_enc.authTag,
1272                        info->cipher.aesgcm_enc.authTagSz,
1273                        info->cipher.aesgcm_enc.authIn,
1274                        info->cipher.aesgcm_enc.authInSz);
1275            }
1276            else {
1277                Aes* aes = info->cipher.aesgcm_dec.aes;
1278                if (aes == NULL)
1279                    return BAD_FUNC_ARG;
1280
1281                rc = IntelQaSymAesGcmDecrypt(dev,
1282                        info->cipher.aesgcm_dec.out,
1283                        info->cipher.aesgcm_dec.in,
1284                        info->cipher.aesgcm_dec.sz,
1285                        (const byte*)aes->devKey, aes->keylen,
1286                        info->cipher.aesgcm_dec.iv,
1287                        info->cipher.aesgcm_dec.ivSz,
1288                        info->cipher.aesgcm_dec.authTag,
1289                        info->cipher.aesgcm_dec.authTagSz,
1290                        info->cipher.aesgcm_dec.authIn,
1291                        info->cipher.aesgcm_dec.authInSz);
1292            }
1293        }
1294        #endif /* HAVE_AESGCM */
1295
1296        #ifndef NO_DES3
1297        if (info->cipher.type == WC_CIPHER_DES3) {
1298            Des3* des = info->cipher.des3.des;
1299            if (des == NULL)
1300                return BAD_FUNC_ARG;
1301
1302            if (info->cipher.enc) {
1303                rc = IntelQaSymDes3CbcEncrypt(dev,
1304                        info->cipher.des3.out,
1305                        info->cipher.des3.in,
1306                        info->cipher.des3.sz,
1307                        (byte*)des->devKey, DES3_KEYLEN,
1308                        (byte*)des->reg, DES_BLOCK_SIZE);
1309            }
1310            else {
1311                rc = IntelQaSymDes3CbcDecrypt(dev,
1312                        info->cipher.des3.out,
1313                        info->cipher.des3.in,
1314                        info->cipher.des3.sz,
1315                        (byte*)des->devKey, DES3_KEYLEN,
1316                        (byte*)des->reg, DES_BLOCK_SIZE);
1317            }
1318        }
1319        #endif /* !NO_DES3 */
1320    }
1321    #endif /* QAT_ENABLE_CRYPTO */
1322
1323    return rc;
1324}
1325
1326/* -------------------------------------------------------------------------- */
1327/* Public API                                                                 */
1328/* -------------------------------------------------------------------------- */
1329
1330int wc_CryptoCb_InitIntelQa(void)
1331{
1332    int devId, rc;
1333
1334    devId = IntelQaInit(NULL);
1335    if (devId < 0) {
1336        QLOG("Couldn't init the Intel QA\n");
1337        devId = INVALID_DEVID;
1338    }
1339    else {
1340        rc = IntelQaOpen(&qaDev, devId);
1341        if (rc != 0) {
1342            QLOG("Couldn't open the device\n");
1343            IntelQaDeInit(devId);
1344            devId = INVALID_DEVID;
1345        }
1346        else {
1347            rc = wc_CryptoCb_RegisterDevice(devId,
1348                    IntelQaSymSync_CryptoDevCb, &qaDev);
1349            if (rc != 0) {
1350                QLOG("Couldn't register the device\n");
1351                IntelQaClose(&qaDev);
1352                IntelQaDeInit(devId);
1353                devId = INVALID_DEVID;
1354            }
1355        }
1356    }
1357
1358    return devId;
1359}
1360
1361
1362void wc_CryptoCb_CleanupIntelQa(int* id)
1363{
1364    if (INVALID_DEVID != *id) {
1365        wc_CryptoCb_UnRegisterDevice(*id);
1366        IntelQaClose(&qaDev);
1367        IntelQaDeInit(*id);
1368        *id = INVALID_DEVID;
1369    }
1370}
1371
1372#endif /* WOLF_CRYPTO_CB */
1373
1374
1375/* -------------------------------------------------------------------------- */
1376/* Memory allocator and deallocator                                           */
1377/* -------------------------------------------------------------------------- */
1378#include <stdio.h>
1379#include <stdlib.h>
1380#include <string.h>
1381
1382/* use thread local for QAE variables (removing mutex requirement) */
1383#ifdef USE_QAE_THREAD_LS
1384    #include <pthread.h> /* for threadId tracking */
1385    #define QAE_THREAD_LS THREAD_LS_T
1386#else
1387    #define QAE_THREAD_LS
1388#endif
1389
1390/* these are used to align memory to a byte boundary */
1391#define ALIGNMENT_BASE     (16ul)
1392#define ALIGNMENT_HW       (64ul)
1393#define WOLF_MAGIC_NUM      0xA576F6C6641736EBUL /* (0xA)WolfAsyn(0xB) */
1394#define WOLF_HEADER_ALIGN   ALIGNMENT_BASE
1395
1396#define QAE_NOT_NUMA_PAGE 0xFFFF
1397typedef struct qaeMemHeader {
1398#ifdef WOLFSSL_TRACK_MEMORY
1399    struct qaeMemHeader* next;
1400    struct qaeMemHeader* prev;
1401    #ifdef WOLFSSL_DEBUG_MEMORY
1402        const char* func;
1403        unsigned int line;
1404    #endif
1405#endif
1406    uint64_t magic;
1407    void* heap;
1408#ifdef USE_QAE_THREAD_LS
1409    pthread_t threadId;
1410#endif
1411    size_t size;
1412    word16 count;
1413    word16 isNuma:1;
1414    word16 reservedBits:15; /* use for future bits */
1415    word16 type;
1416    word16 numa_page_offset; /* use QAE_NOT_NUMA_PAGE if not NUMA */
1417} ALIGN16 qaeMemHeader;
1418
1419#ifdef WOLFSSL_TRACK_MEMORY
1420    typedef struct qaeMemStats {
1421        long totalAllocs;     /* number of allocations */
1422        long totalDeallocs;   /* number of deallocations */
1423        long totalBytes;      /* total number of bytes allocated */
1424        long peakBytes;       /* concurrent max bytes */
1425        long currentBytes;    /* total current bytes in use */
1426    } qaeMemStats;
1427
1428    /* track allocations and report at end */
1429    typedef struct qaeMemList {
1430        qaeMemHeader* head;
1431        qaeMemHeader* tail;
1432        uint32_t count;
1433    } qaeMemList;
1434#endif /* WOLFSSL_TRACK_MEMORY */
1435
1436
1437/* local variables */
1438#ifndef USE_QAE_THREAD_LS
1439    static pthread_mutex_t g_memLock = PTHREAD_MUTEX_INITIALIZER;
1440#endif
1441
1442
1443#ifdef WOLFSSL_TRACK_MEMORY
1444    static qaeMemStats g_memStats;
1445    static qaeMemList g_memList;
1446    static pthread_mutex_t g_memStatLock = PTHREAD_MUTEX_INITIALIZER;
1447#endif
1448
1449static WC_INLINE int qaeMemTypeIsNuma(int type)
1450{
1451    int isNuma = 0;
1452
1453    switch (type) {
1454        case DYNAMIC_TYPE_ASYNC_NUMA:
1455        case DYNAMIC_TYPE_ASYNC_NUMA64:
1456        case DYNAMIC_TYPE_WOLF_BIGINT:
1457        case DYNAMIC_TYPE_PRIVATE_KEY:
1458        case DYNAMIC_TYPE_PUBLIC_KEY:
1459        case DYNAMIC_TYPE_AES_BUFFER:
1460        case DYNAMIC_TYPE_RSA_BUFFER:
1461        case DYNAMIC_TYPE_ECC_BUFFER:
1462        case DYNAMIC_TYPE_SIGNATURE:
1463        case DYNAMIC_TYPE_DIGEST:
1464        case DYNAMIC_TYPE_SECRET:
1465        case DYNAMIC_TYPE_SEED:
1466        case DYNAMIC_TYPE_SALT:
1467        {
1468            isNuma = 1;
1469            break;
1470        }
1471        case DYNAMIC_TYPE_OUT_BUFFER:
1472        case DYNAMIC_TYPE_IN_BUFFER:
1473        {
1474        #if !defined(WC_ASYNC_NO_CRYPT) && !defined(WC_ASYNC_NO_HASH)
1475            isNuma = 1;
1476        #else
1477            isNuma = 0;
1478        #endif
1479            break;
1480        }
1481        default:
1482            isNuma = 0;
1483            break;
1484    }
1485    return isNuma;
1486}
1487
1488
1489static void _qaeMemFree(void *ptr, void* heap, int type
1490#ifdef WOLFSSL_DEBUG_MEMORY
1491    , const char* func, unsigned int line
1492#endif
1493)
1494{
1495    qaeMemHeader* header = NULL;
1496    size_t size;
1497    void* origPtr = ptr;
1498
1499    if (ptr == NULL)
1500        return;
1501
1502    /* adjust for header and align */
1503    ptr = (byte*)(((size_t)ptr - ((size_t)ptr % WOLF_HEADER_ALIGN)) -
1504        sizeof(qaeMemHeader));
1505    header = (qaeMemHeader*)ptr;
1506
1507    /* check for header magic */
1508    if (header->magic != WOLF_MAGIC_NUM) {
1509        printf("Free: Header magic not found! %p\n", ptr);
1510        return;
1511    }
1512
1513    /* cache values for later */
1514    size = header->size;
1515
1516#ifdef WOLFSSL_DEBUG_MEMORY
1517#ifdef WOLFSSL_DEBUG_MEMORY_PRINT
1518    printf("Free: %p (%u) at %s:%u, heap %p, type %d, count %d\n",
1519        origPtr, (unsigned int)size, func, line, heap, type, header->count);
1520#else
1521    (void)func;
1522    (void)line;
1523#endif
1524#endif
1525    (void)type;
1526
1527    /* adjust free count */
1528    header->count--;
1529
1530    /* check header count */
1531    if (header->count > 0) {
1532        /* go ahead and return if still in use */
1533        return;
1534    }
1535
1536#ifdef WOLFSSL_TRACK_MEMORY
1537    if (pthread_mutex_lock(&g_memStatLock) == 0) {
1538        g_memStats.currentBytes -= size;
1539        g_memStats.totalDeallocs++;
1540
1541        if (header == g_memList.head && header == g_memList.tail) {
1542            g_memList.head = NULL;
1543            g_memList.tail = NULL;
1544        }
1545        else if (header == g_memList.head) {
1546            g_memList.head = header->next;
1547            g_memList.head->prev = NULL;
1548        }
1549        else if (header == g_memList.tail) {
1550            g_memList.tail = header->prev;
1551            g_memList.tail->next = NULL;
1552        }
1553        else {
1554            qaeMemHeader* next = header->next;
1555            qaeMemHeader* prev = header->prev;
1556            if (next)
1557                next->prev = prev;
1558            if (prev)
1559                prev->next = next;
1560        }
1561        g_memList.count--;
1562
1563        pthread_mutex_unlock(&g_memStatLock);
1564    }
1565#endif
1566
1567    (void)heap;
1568    (void)size;
1569    (void)origPtr;
1570
1571#ifdef WOLFSSL_DEBUG_MEMORY
1572    /* make sure magic is gone */
1573    header->magic = 0;
1574#endif
1575
1576    /* free type */
1577    if (header->isNuma && header->numa_page_offset != QAE_NOT_NUMA_PAGE) {
1578        qaeMemFreeNUMA(&ptr);
1579    }
1580    else {
1581        free(ptr);
1582    }
1583}
1584
1585
1586static void* _qaeMemAlloc(size_t size, void* heap, int type
1587#ifdef WOLFSSL_DEBUG_MEMORY
1588    , const char* func, unsigned int line
1589#endif
1590)
1591{
1592    void* ptr = NULL;
1593    qaeMemHeader* header = NULL;
1594    int isNuma;
1595    int alignment = ALIGNMENT_BASE;
1596    word16 page_offset = QAE_NOT_NUMA_PAGE;
1597
1598    /* make sure all allocations are aligned */
1599    if ((size % WOLF_HEADER_ALIGN) != 0) {
1600        size += (WOLF_HEADER_ALIGN - (size % WOLF_HEADER_ALIGN));
1601    }
1602
1603    isNuma = qaeMemTypeIsNuma(type);
1604    if (type == DYNAMIC_TYPE_ASYNC_NUMA64)
1605        alignment = ALIGNMENT_HW;
1606
1607    /* allocate type */
1608    if (isNuma) {
1609        /* Node is typically 0 */
1610        page_offset = 0;
1611        ptr = qaeMemAllocNUMA((Cpa32U)(size + sizeof(qaeMemHeader)), 0,
1612            alignment);
1613    }
1614    else {
1615        isNuma = 0;
1616        ptr = malloc(size + sizeof(qaeMemHeader));
1617    }
1618
1619    /* add header */
1620    if (ptr) {
1621        header = (qaeMemHeader*)ptr;
1622        ptr = (byte*)ptr + sizeof(qaeMemHeader);
1623        header->magic = WOLF_MAGIC_NUM;
1624        header->heap = heap;
1625        header->size = size;
1626        header->type = type;
1627        header->count = 1;
1628        header->isNuma = isNuma;
1629        header->numa_page_offset = page_offset;
1630    #ifdef USE_QAE_THREAD_LS
1631        header->threadId = pthread_self();
1632    #endif
1633
1634    #ifdef WOLFSSL_TRACK_MEMORY
1635        if (pthread_mutex_lock(&g_memStatLock) == 0) {
1636            g_memStats.totalAllocs++;
1637            g_memStats.totalBytes   += size;
1638            g_memStats.currentBytes += size;
1639            if (g_memStats.currentBytes > g_memStats.peakBytes)
1640                g_memStats.peakBytes = g_memStats.currentBytes;
1641
1642        #ifdef WOLFSSL_DEBUG_MEMORY
1643            header->func = func;
1644            header->line = line;
1645        #endif
1646
1647            /* Setup event */
1648            header->next = NULL;
1649            if (g_memList.tail == NULL)  {
1650                g_memList.head = header;
1651            }
1652            else {
1653                g_memList.tail->next = header;
1654                header->prev = g_memList.tail;
1655            }
1656            g_memList.tail = header;      /* add to the end either way */
1657            g_memList.count++;
1658
1659            pthread_mutex_unlock(&g_memStatLock);
1660        }
1661    #endif
1662    }
1663
1664#ifdef WOLFSSL_DEBUG_MEMORY
1665#ifdef WOLFSSL_DEBUG_MEMORY_PRINT
1666    printf("Alloc: %p (%u) at %s:%u, heap %p, type %d\n",
1667        ptr, (unsigned int)size, func, line, heap, type);
1668#else
1669    (void)func;
1670    (void)line;
1671#endif
1672#endif
1673
1674    (void)heap;
1675
1676    return ptr;
1677}
1678
1679/* Public Functions */
1680void* wc_CryptoCb_IntelQaMalloc(size_t size, void* heap, int type
1681#ifdef WOLFSSL_DEBUG_MEMORY
1682    , const char* func, unsigned int line
1683#endif
1684)
1685{
1686    void* ptr;
1687
1688#ifndef USE_QAE_THREAD_LS
1689    int ret = pthread_mutex_lock(&g_memLock);
1690    if (ret != 0) {
1691        printf("Alloc: Error(%d) on mutex lock\n", ret);
1692        return NULL;
1693    }
1694#endif
1695
1696    ptr = _qaeMemAlloc(size, heap, type
1697    #ifdef WOLFSSL_DEBUG_MEMORY
1698        , func, line
1699    #endif
1700    );
1701
1702#ifndef USE_QAE_THREAD_LS
1703    pthread_mutex_unlock(&g_memLock);
1704#endif
1705
1706    return ptr;
1707}
1708
1709void wc_CryptoCb_IntelQaFree(void *ptr, void* heap, int type
1710#ifdef WOLFSSL_DEBUG_MEMORY
1711    , const char* func, unsigned int line
1712#endif
1713)
1714{
1715#ifndef USE_QAE_THREAD_LS
1716    int ret = pthread_mutex_lock(&g_memLock);
1717    if (ret != 0) {
1718        printf("Free: Error(%d) on mutex lock\n", ret);
1719        return;
1720    }
1721#endif
1722
1723    _qaeMemFree(ptr, heap, type
1724    #ifdef WOLFSSL_DEBUG_MEMORY
1725        , func, line
1726    #endif
1727    );
1728
1729#ifndef USE_QAE_THREAD_LS
1730    pthread_mutex_unlock(&g_memLock);
1731#endif
1732}
1733
1734void* wc_CryptoCb_IntelQaRealloc(void *ptr, size_t size, void* heap, int type
1735#ifdef WOLFSSL_DEBUG_MEMORY
1736    , const char* func, unsigned int line
1737#endif
1738)
1739{
1740    void* newPtr = NULL;
1741    void* origPtr = ptr;
1742    qaeMemHeader* header = NULL;
1743    byte allocNew = 1;
1744    int newIsNuma = -1, ptrIsNuma = -1;
1745    size_t copySize = 0;
1746
1747#ifndef USE_QAE_THREAD_LS
1748    int ret = pthread_mutex_lock(&g_memLock);
1749    if (ret != 0) {
1750        printf("Realloc: Error(%d) on mutex lock\n", ret);
1751        return NULL;
1752    }
1753#endif
1754
1755    (void)heap;
1756
1757    if (ptr) {
1758        /* get header pointer and align */
1759        header = (qaeMemHeader*)(((size_t)ptr -
1760            ((size_t)ptr % WOLF_HEADER_ALIGN)) - sizeof(qaeMemHeader));
1761        if (header->magic == WOLF_MAGIC_NUM) {
1762            newIsNuma = qaeMemTypeIsNuma(type);
1763            ptrIsNuma = (header->numa_page_offset != QAE_NOT_NUMA_PAGE) ? 1 : 0;
1764
1765            /* for non-NUMA, treat as normal REALLOC */
1766            if (newIsNuma == 0 && ptrIsNuma == 0) {
1767                allocNew = 1;
1768            }
1769            /* confirm input is aligned, otherwise allocate new */
1770            else if (((size_t)ptr % WOLF_HEADER_ALIGN) != 0) {
1771                allocNew = 1;
1772            }
1773            /* if matching NUMA type and size fits, use existing */
1774            else if (newIsNuma == ptrIsNuma && header->size >= size) {
1775
1776            #ifdef USE_QAE_THREAD_LS
1777                if (header->threadId != pthread_self()) {
1778                    allocNew = 1;
1779                #if 0
1780                    printf("Realloc %p from different thread! orig %lx this %lx\n",
1781                        origPtr, header->threadId, pthread_self());
1782                #endif
1783                }
1784                else
1785            #endif
1786                {
1787                    /* use existing pointer and increment counter */
1788                    header->count++;
1789                    newPtr = origPtr;
1790                    allocNew = 0;
1791                }
1792            }
1793
1794            copySize = header->size;
1795        }
1796        else {
1797            copySize = size;
1798        }
1799    }
1800
1801    if (allocNew) {
1802        newPtr = _qaeMemAlloc(size, heap, type
1803        #ifdef WOLFSSL_DEBUG_MEMORY
1804            , func, line
1805        #endif
1806        );
1807        if (newPtr && ptr) {
1808            /* only copy min of new and old size to new pointer */
1809            if (copySize > size)
1810                copySize = size;
1811            XMEMCPY(newPtr, ptr, copySize);
1812
1813            if (newIsNuma == 0 && ptrIsNuma == 0) {
1814                /* for non-NUMA, treat as normal REALLOC and free old pointer */
1815                _qaeMemFree(ptr, heap, type
1816                #ifdef WOLFSSL_DEBUG_MEMORY
1817                    , func, line
1818                #endif
1819                );
1820            }
1821        }
1822    }
1823
1824#ifndef USE_QAE_THREAD_LS
1825    pthread_mutex_unlock(&g_memLock);
1826#endif
1827
1828#ifdef WOLFSSL_DEBUG_MEMORY
1829#ifdef WOLFSSL_DEBUG_MEMORY_PRINT
1830    if (allocNew) {
1831        printf("Realloc: New %p -> %p (%u) at %s:%u, heap %p, type %d\n",
1832            origPtr, newPtr, (unsigned int)size, func, line, heap, type);
1833    }
1834    else {
1835        printf("Realloc: Reuse %p (%u) at %s:%u, heap %p, type %d, count %d\n",
1836             origPtr, (unsigned int)size, func, line, header->heap, header->type, header->count);
1837    }
1838#else
1839    (void)func;
1840    (void)line;
1841#endif
1842#endif
1843
1844    return newPtr;
1845}
1846
1847
1848#ifdef WOLFSSL_TRACK_MEMORY
1849int InitMemoryTracker(void)
1850{
1851    if (pthread_mutex_lock(&g_memStatLock) == 0) {
1852        g_memStats.totalAllocs  = 0;
1853        g_memStats.totalDeallocs= 0;
1854        g_memStats.totalBytes   = 0;
1855        g_memStats.peakBytes    = 0;
1856        g_memStats.currentBytes = 0;
1857
1858        XMEMSET(&g_memList, 0, sizeof(g_memList));
1859
1860        pthread_mutex_unlock(&g_memStatLock);
1861    }
1862
1863    return 0;
1864}
1865
1866void ShowMemoryTracker(void)
1867{
1868    if (pthread_mutex_lock(&g_memStatLock) == 0) {
1869        printf("total   Allocs = %9ld\n", g_memStats.totalAllocs);
1870        printf("total Deallocs = %9ld\n", g_memStats.totalDeallocs);
1871        printf("total    Bytes = %9ld\n", g_memStats.totalBytes);
1872        printf("peak     Bytes = %9ld\n", g_memStats.peakBytes);
1873        printf("current  Bytes = %9ld\n", g_memStats.currentBytes);
1874
1875        if (g_memList.count > 0) {
1876
1877            /* print list of allocations */
1878            qaeMemHeader* header;
1879            for (header = g_memList.head; header != NULL; header = header->next) {
1880                printf("Leak: Ptr %p, Size %u, Type %d, Heap %p"
1881                #ifdef WOLFSSL_DEBUG_MEMORY
1882                    ", Func %s, Line %d"
1883                #endif
1884                    "\n",
1885                    (byte*)header + sizeof(qaeMemHeader), (unsigned int)header->size,
1886                    header->type, header->heap
1887                #ifdef WOLFSSL_DEBUG_MEMORY
1888                    , header->func, header->line
1889                #endif
1890                );
1891            }
1892        }
1893
1894        pthread_mutex_unlock(&g_memStatLock);
1895
1896        /* cleanup lock */
1897        pthread_mutex_destroy(&g_memStatLock);
1898    }
1899}
1900#endif /* WOLFSSL_TRACK_MEMORY */
1901
1902#ifdef QAT_DEMO_MAIN
1903
1904/* AES GCM */
1905static const byte aesgcm_k[] = {
1906    0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88,
1907    0x99, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66,
1908    0x77, 0x88, 0x99, 0x00, 0x11, 0x22, 0x33, 0x44,
1909    0x55, 0x66, 0x77, 0x88, 0x99, 0x00, 0x11, 0x22
1910};
1911
1912static const byte aesgcm_iv[] = {
1913    0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe, 0xca, 0xfe,
1914    0xca, 0xfe, 0xca, 0xfe
1915};
1916
1917static const byte aesgcm_a[] = {
1918    0xde, 0xad, 0xde, 0xad, 0xde, 0xad, 0xde, 0xad,
1919    0xde, 0xad, 0xde, 0xad, 0xde, 0xad, 0xde, 0xad,
1920    0xde, 0xad, 0xde, 0xad
1921};
1922
1923static const byte aesgcm_p[] = {
1924    0x79, 0x84, 0x86, 0x44, 0x68, 0x45, 0x15, 0x61,
1925    0x86, 0x54, 0x66, 0x56, 0x54, 0x54, 0x31, 0x54,
1926    0x64, 0x64, 0x68, 0x45, 0x15, 0x15, 0x61, 0x61,
1927    0x51, 0x51, 0x51, 0x51, 0x51, 0x56, 0x14, 0x11,
1928    0x72, 0x13, 0x51, 0x82, 0x84, 0x56, 0x74, 0x53,
1929    0x45, 0x34, 0x65, 0x15, 0x46, 0x14, 0x67, 0x55,
1930    0x16, 0x14, 0x67, 0x54, 0x65, 0x47, 0x14, 0x67,
1931    0x46, 0x74, 0x65, 0x46
1932};
1933
1934static const byte aesgcm_c[] = {
1935    0x59, 0x85, 0x02, 0x97, 0xE0, 0x4D, 0xFC, 0x5C,
1936    0x03, 0xCC, 0x83, 0x64, 0xCE, 0x28, 0x0B, 0x95,
1937    0x78, 0xEC, 0x93, 0x40, 0xA1, 0x8D, 0x21, 0xC5,
1938    0x48, 0x6A, 0x39, 0xBA, 0x4F, 0x4B, 0x8C, 0x95,
1939    0x6F, 0x8C, 0xF6, 0x9C, 0xD0, 0xA5, 0x8D, 0x67,
1940    0xA1, 0x32, 0x11, 0xE7, 0x2E, 0xF6, 0x63, 0xAF,
1941    0xDE, 0xD4, 0x7D, 0xEC, 0x15, 0x01, 0x58, 0xCB,
1942    0xE3, 0x7B, 0xC6, 0x94,
1943};
1944
1945static byte aesgcm_t[] = {
1946    0x5D, 0x10, 0x3F, 0xC7, 0x22, 0xC7, 0x21, 0x29
1947};
1948
1949
1950/* simple example of using AES-GCM encrypt with Intel QA */
1951int main(int argc, char** argv)
1952{
1953#if !defined(NO_AES) && defined(HAVE_AESGCM)
1954    int ret;
1955    IntelQaDev dev;
1956    byte out[256];
1957    byte tmp[256];
1958    word32 tmpLen;
1959#endif
1960
1961#ifdef QAT_DEBUG
1962    wolfSSL_Debugging_ON();
1963#endif
1964
1965    IntelQaInit(NULL);
1966
1967#ifndef NO_AES
1968#ifdef HAVE_AESGCM
1969    /* AES Test */
1970    IntelQaOpen(&dev, 0);
1971    dev.event.ret = WC_PENDING_E;
1972    tmpLen = sizeof(aesgcm_t);
1973    XMEMSET(out, 0, sizeof(out));
1974    XMEMSET(tmp, 0, sizeof(tmp));
1975
1976    ret = IntelQaSymAesGcmEncrypt(&dev, out, aesgcm_p, sizeof(aesgcm_p),
1977        aesgcm_k, sizeof(aesgcm_k), aesgcm_iv, sizeof(aesgcm_iv),
1978        tmp, tmpLen, aesgcm_a, sizeof(aesgcm_a));
1979    printf("AES GCM Encrypt: Ret=%d, Tag Len=%d\n", ret, tmpLen);
1980    IntelQaClose(&dev);
1981#endif /* HAVE_AESGCM */
1982#endif /* NO_AES */
1983
1984    IntelQaDeInit(0);
1985
1986    return 0;
1987}
1988
1989#endif
1990
1991#endif /* HAVE_INTEL_QA_SYNC */