/* quickassist_mem.c * * Copyright (C) 2006-2026 wolfSSL Inc. * * This file is part of wolfSSL. * * wolfSSL is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 3 of the License, or * (at your option) any later version. * * wolfSSL is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA */ #ifdef HAVE_CONFIG_H #include #endif #include #ifdef HAVE_INTEL_QA #include #include #include #include #include #include /* use thread local for QAE variables (removing mutex requirement) */ #include /* for threadId tracking */ #ifdef USE_QAE_THREAD_LS #define QAE_THREAD_LS THREAD_LS_T #else #define QAE_THREAD_LS #endif /* these are used to align memory to a byte boundary */ #define ALIGNMENT_BASE (16ul) #define ALIGNMENT_HW (64ul) #define WOLF_MAGIC_NUM 0xA576F6C6641736EBUL /* (0xA)WolfAsyn(0xB) */ #define WOLF_HEADER_ALIGN ALIGNMENT_BASE #ifndef QAT_V2 #include #include #include #include #include #include #include #include #ifdef SAL_IOMMU_CODE #include #endif /* enable fixed static memory instead of dynamic list */ #ifdef USE_QAE_STATIC_MEM /* adjustable parameter for the maximum memory allocations */ #ifndef QAE_USER_MEM_MAX_COUNT #define QAE_USER_MEM_MAX_COUNT 16000 #endif #define MEM_INVALID_IDX -1 #endif #define QAE_MEM "/dev/qae_mem" #define PAGE_SHIFT 13 #define PAGE_SIZE (1UL << PAGE_SHIFT) #define PAGE_MASK (~(PAGE_SIZE-1)) #define SYSTEM_PAGE_SHIFT 12 #define SYSTEM_PAGE_SIZE (1UL << SYSTEM_PAGE_SHIFT) #define SYSTEM_PAGE_MASK (~(SYSTEM_PAGE_SIZE-1)) #define USER_MEM_OFFSET (128) #define QAEM_MAGIC_NUM 0xABCD12345678ECDFUL /* define types which need to vary between 32 and 64 bit */ #ifdef __x86_64__ #define QAE_UINT Cpa64U #define QAE_INT Cpa64S #else #define QAE_UINT Cpa32U #define QAE_INT Cpa32S #endif /* IOCTL number for use between the kernel and the user space application */ #define DEV_MEM_MAGIC 'q' #define DEV_MEM_CMD_MEMALLOC (0) #define DEV_MEM_CMD_MEMFREE (1) /* IOCTL commands for requesting kernel memory */ #define DEV_MEM_IOC_MEMALLOC \ _IOWR(DEV_MEM_MAGIC, DEV_MEM_CMD_MEMALLOC, qae_dev_mem_info_t) #define DEV_MEM_IOC_MEMFREE \ _IOWR(DEV_MEM_MAGIC, DEV_MEM_CMD_MEMFREE, qae_dev_mem_info_t) /* local structures */ #pragma pack(push) #pragma pack(1) typedef struct qae_dev_mem_info_s { union { struct qae_dev_mem_info_s *pPrev; uint64_t padding_pPrev; }; union { struct qae_dev_mem_info_s *pNext; uint64_t padding_pNext; }; uint32_t id; /* Id of this block */ uint32_t nodeId; /* Node id for NUMA */ uint32_t size; /* Size of this block (bytes) */ uint32_t available_size; /* Available size remained on the page */ uint16_t allocations; /* Counter keeping track of number of allocations */ union { void *kmalloc_ptr; uint64_t padding_kmalloc_ptr; }; /* Pointer to mem originally returned by kmalloc */ union { int32_t *kmalloc_area; uint64_t padding_kamalloc_area; }; /* Pointer to kmalloc'd area rounded up to a page boundary */ uint64_t phy_addr; /* Physical address of the kmalloc area */ union { void *virt_addr; uint64_t padding_virt_addr; }; /* Base address in user space - i.e. virtual address */ } qae_dev_mem_info_t; #ifdef USE_QAE_STATIC_MEM typedef struct qae_dev_mem_info_ex_s { qae_dev_mem_info_t mem_info; int index; /* Index into g_pUserMemList */ } qae_dev_mem_info_ex_t; #else typedef qae_dev_mem_info_t qae_dev_mem_info_ex_t; #endif #pragma pack(pop) #endif /* QAT_V2 */ #define QAE_NOT_NUMA_PAGE 0xFFFF typedef struct qaeMemHeader { #ifdef WOLFSSL_TRACK_MEMORY struct qaeMemHeader* next; struct qaeMemHeader* prev; #ifdef WOLFSSL_DEBUG_MEMORY const char* func; unsigned int line; #endif #endif uint64_t magic; void* heap; #ifdef USE_QAE_THREAD_LS pthread_t threadId; #endif size_t size; word16 count; word16 isNuma:1; word16 reservedBits:15; /* use for future bits */ word16 type; word16 numa_page_offset; /* use QAE_NOT_NUMA_PAGE if not NUMA */ } ALIGN16 qaeMemHeader; #ifdef WOLFSSL_TRACK_MEMORY typedef struct qaeMemStats { long totalAllocs; /* number of allocations */ long totalDeallocs; /* number of deallocations */ long totalBytes; /* total number of bytes allocated */ long peakBytes; /* concurrent max bytes */ long currentBytes; /* total current bytes in use */ } qaeMemStats; /* track allocations and report at end */ typedef struct qaeMemList { qaeMemHeader* head; qaeMemHeader* tail; uint32_t count; } qaeMemList; #endif /* WOLFSSL_TRACK_MEMORY */ /* local variables */ #ifndef USE_QAE_THREAD_LS static pthread_mutex_t g_memLock = PTHREAD_MUTEX_INITIALIZER; #endif #ifndef QAT_V2 #ifdef USE_QAE_STATIC_MEM /* Use an array instead of a list */ static QAE_THREAD_LS qae_dev_mem_info_ex_t* g_pUserMemList[QAE_USER_MEM_MAX_COUNT]; /* cache the available sizes to improve userMemLookupBySize performance */ static QAE_THREAD_LS uint16_t g_avail_size[QAE_USER_MEM_MAX_COUNT]; /* Count of items in g_pUserMemList and g_avail_size */ static QAE_THREAD_LS int g_userMemListCount = 0; static QAE_THREAD_LS int g_lastIndexBySize = 0; #else static QAE_THREAD_LS qae_dev_mem_info_t *g_pUserMemList = NULL; static QAE_THREAD_LS qae_dev_mem_info_t *g_pUserMemListHead = NULL; #endif static int g_qaeMemFd = -1; #endif /* !QAT_V2 */ #ifdef WOLFSSL_TRACK_MEMORY static qaeMemStats g_memStats; static qaeMemList g_memList; static pthread_mutex_t g_memStatLock = PTHREAD_MUTEX_INITIALIZER; #endif /* forward declarations */ #ifndef QAT_V2 static void* qaeMemAllocNUMA(Cpa32U size, Cpa32U node, Cpa32U alignment, word16* p_page_offset); static void qaeMemFreeNUMA(void** ptr, word16 page_offset); #endif static WC_INLINE int qaeMemTypeIsNuma(int type) { int isNuma = 0; switch (type) { case DYNAMIC_TYPE_ASYNC_NUMA: case DYNAMIC_TYPE_ASYNC_NUMA64: case DYNAMIC_TYPE_WOLF_BIGINT: case DYNAMIC_TYPE_PRIVATE_KEY: case DYNAMIC_TYPE_PUBLIC_KEY: case DYNAMIC_TYPE_AES_BUFFER: case DYNAMIC_TYPE_RSA_BUFFER: case DYNAMIC_TYPE_ECC_BUFFER: case DYNAMIC_TYPE_SIGNATURE: case DYNAMIC_TYPE_DIGEST: case DYNAMIC_TYPE_SECRET: case DYNAMIC_TYPE_SEED: case DYNAMIC_TYPE_SALT: { isNuma = 1; break; } case DYNAMIC_TYPE_OUT_BUFFER: case DYNAMIC_TYPE_IN_BUFFER: { #if !defined(WC_ASYNC_NO_CRYPT) && !defined(WC_ASYNC_NO_HASH) isNuma = 1; #else isNuma = 0; #endif break; } default: isNuma = 0; break; } return isNuma; } static void _qaeMemFree(void *ptr, void* heap, int type #ifdef WOLFSSL_DEBUG_MEMORY , const char* func, unsigned int line #endif ) { qaeMemHeader* header = NULL; size_t size; void* origPtr = ptr; if (ptr == NULL) return; /* adjust for header and align */ ptr = (byte*)(((size_t)ptr - ((size_t)ptr % WOLF_HEADER_ALIGN)) - sizeof(qaeMemHeader)); header = (qaeMemHeader*)ptr; /* check for header magic */ if (header->magic != WOLF_MAGIC_NUM) { printf("Free: Header magic not found! %p\n", ptr); return; } /* cache values for later */ size = header->size; #ifdef WOLFSSL_DEBUG_MEMORY #ifdef WOLFSSL_DEBUG_MEMORY_PRINT printf("Free: %p (%u) at %s:%d, heap %p, type %d, count %d\n", origPtr, (unsigned int)size, func, line, heap, type, header->count); #else (void)func; (void)line; #endif #endif (void)type; /* adjust free count */ header->count--; /* check header count */ if (header->count > 0) { /* go ahead and return if still in use */ return; } #ifdef WOLFSSL_TRACK_MEMORY if (pthread_mutex_lock(&g_memStatLock) == 0) { g_memStats.currentBytes -= size; g_memStats.totalDeallocs++; if (header == g_memList.head && header == g_memList.tail) { g_memList.head = NULL; g_memList.tail = NULL; } else if (header == g_memList.head) { g_memList.head = header->next; g_memList.head->prev = NULL; } else if (header == g_memList.tail) { g_memList.tail = header->prev; g_memList.tail->next = NULL; } else { qaeMemHeader* next = header->next; qaeMemHeader* prev = header->prev; if (next) next->prev = prev; if (prev) prev->next = next; } g_memList.count--; pthread_mutex_unlock(&g_memStatLock); } #endif (void)heap; (void)size; (void)origPtr; #ifdef WOLFSSL_DEBUG_MEMORY /* make sure magic is gone */ header->magic = 0; #endif /* free type */ if (header->isNuma && header->numa_page_offset != QAE_NOT_NUMA_PAGE) { #ifdef QAT_V2 qaeMemFreeNUMA(&ptr); #else qaeMemFreeNUMA(&ptr, header->numa_page_offset); #endif } else { free(ptr); } } static void* _qaeMemAlloc(size_t size, void* heap, int type #ifdef WOLFSSL_DEBUG_MEMORY , const char* func, unsigned int line #endif ) { void* ptr = NULL; qaeMemHeader* header = NULL; int isNuma; int alignment = ALIGNMENT_BASE; word16 page_offset = QAE_NOT_NUMA_PAGE; /* make sure all allocations are aligned */ if ((size % WOLF_HEADER_ALIGN) != 0) { size += (WOLF_HEADER_ALIGN - (size % WOLF_HEADER_ALIGN)); } isNuma = qaeMemTypeIsNuma(type); if (type == DYNAMIC_TYPE_ASYNC_NUMA64) alignment = ALIGNMENT_HW; /* allocate type */ if (isNuma) { /* Node is typically 0 */ #ifdef QAT_V2 page_offset = 0; ptr = qaeMemAllocNUMA((Cpa32U)(size + sizeof(qaeMemHeader)), 0, alignment); #else ptr = qaeMemAllocNUMA((Cpa32U)(size + sizeof(qaeMemHeader)), 0, alignment, &page_offset); #endif } else { isNuma = 0; ptr = malloc(size + sizeof(qaeMemHeader)); } /* add header */ if (ptr) { header = (qaeMemHeader*)ptr; ptr = (byte*)ptr + sizeof(qaeMemHeader); header->magic = WOLF_MAGIC_NUM; header->heap = heap; header->size = size; header->type = type; header->count = 1; header->isNuma = isNuma; header->numa_page_offset = page_offset; #ifdef USE_QAE_THREAD_LS header->threadId = pthread_self(); #endif #ifdef WOLFSSL_TRACK_MEMORY if (pthread_mutex_lock(&g_memStatLock) == 0) { g_memStats.totalAllocs++; g_memStats.totalBytes += size; g_memStats.currentBytes += size; if (g_memStats.currentBytes > g_memStats.peakBytes) g_memStats.peakBytes = g_memStats.currentBytes; #ifdef WOLFSSL_DEBUG_MEMORY header->func = func; header->line = line; #endif /* Setup event */ header->next = NULL; if (g_memList.tail == NULL) { g_memList.head = header; } else { g_memList.tail->next = header; header->prev = g_memList.tail; } g_memList.tail = header; /* add to the end either way */ g_memList.count++; pthread_mutex_unlock(&g_memStatLock); } #endif } #ifdef WOLFSSL_DEBUG_MEMORY #ifdef WOLFSSL_DEBUG_MEMORY_PRINT printf("Alloc: %p (%u) at %s:%d, heap %p, type %d\n", ptr, (unsigned int)size, func, line, heap, type); #else (void)func; (void)line; #endif #endif (void)heap; return ptr; } /* Public Functions */ void* IntelQaMalloc(size_t size, void* heap, int type #ifdef WOLFSSL_DEBUG_MEMORY , const char* func, unsigned int line #endif ) { void* ptr; #ifndef USE_QAE_THREAD_LS int ret = pthread_mutex_lock(&g_memLock); if (ret != 0) { printf("Alloc: Error(%d) on mutex lock\n", ret); return NULL; } #endif ptr = _qaeMemAlloc(size, heap, type #ifdef WOLFSSL_DEBUG_MEMORY , func, line #endif ); #ifndef USE_QAE_THREAD_LS pthread_mutex_unlock(&g_memLock); #endif return ptr; } void IntelQaFree(void *ptr, void* heap, int type #ifdef WOLFSSL_DEBUG_MEMORY , const char* func, unsigned int line #endif ) { #ifndef USE_QAE_THREAD_LS int ret = pthread_mutex_lock(&g_memLock); if (ret != 0) { printf("Free: Error(%d) on mutex lock\n", ret); return; } #endif _qaeMemFree(ptr, heap, type #ifdef WOLFSSL_DEBUG_MEMORY , func, line #endif ); #ifndef USE_QAE_THREAD_LS pthread_mutex_unlock(&g_memLock); #endif } void* IntelQaRealloc(void *ptr, size_t size, void* heap, int type #ifdef WOLFSSL_DEBUG_MEMORY , const char* func, unsigned int line #endif ) { void* newPtr = NULL; void* origPtr = ptr; qaeMemHeader* header = NULL; byte allocNew = 1; int newIsNuma = -1, ptrIsNuma = -1; size_t copySize = 0; #ifndef USE_QAE_THREAD_LS int ret = pthread_mutex_lock(&g_memLock); if (ret != 0) { printf("Realloc: Error(%d) on mutex lock\n", ret); return NULL; } #endif (void)heap; if (ptr) { /* get header pointer and align */ header = (qaeMemHeader*)(((size_t)ptr - ((size_t)ptr % WOLF_HEADER_ALIGN)) - sizeof(qaeMemHeader)); if (header->magic == WOLF_MAGIC_NUM) { newIsNuma = qaeMemTypeIsNuma(type); ptrIsNuma = (header->numa_page_offset != QAE_NOT_NUMA_PAGE) ? 1 : 0; /* for non-NUMA, treat as normal REALLOC */ if (newIsNuma == 0 && ptrIsNuma == 0) { allocNew = 1; } /* confirm input is aligned, otherwise allocate new */ else if (((size_t)ptr % WOLF_HEADER_ALIGN) != 0) { allocNew = 1; } /* if matching NUMA type and size fits, use existing */ else if (newIsNuma == ptrIsNuma && header->size >= size) { #ifdef USE_QAE_THREAD_LS if (header->threadId != pthread_self()) { allocNew = 1; #if 0 printf("Realloc %p from different thread! " "orig %lx this %lx\n", origPtr, header->threadId, pthread_self()); #endif } else #endif { /* use existing pointer and increment counter */ header->count++; newPtr = origPtr; allocNew = 0; } } copySize = header->size; } else { copySize = size; } } if (allocNew) { newPtr = _qaeMemAlloc(size, heap, type #ifdef WOLFSSL_DEBUG_MEMORY , func, line #endif ); if (newPtr && ptr) { /* only copy min of new and old size to new pointer */ if (copySize > size) copySize = size; XMEMCPY(newPtr, ptr, copySize); if (newIsNuma == 0 && ptrIsNuma == 0) { /* for non-NUMA, treat as normal REALLOC and free old pointer */ _qaeMemFree(ptr, heap, type #ifdef WOLFSSL_DEBUG_MEMORY , func, line #endif ); } } } #ifndef USE_QAE_THREAD_LS pthread_mutex_unlock(&g_memLock); #endif #ifdef WOLFSSL_DEBUG_MEMORY #ifdef WOLFSSL_DEBUG_MEMORY_PRINT if (allocNew) { printf("Realloc: New %p -> %p (%u) at %s:%d, heap %p, type %d\n", origPtr, newPtr, (unsigned int)size, func, line, heap, type); } else { printf("Realloc: Reuse %p (%u) at %s:%d, heap %p, type %d, count %d\n", origPtr, (unsigned int)size, func, line, header->heap, header->type, header->count); } #else (void)func; (void)line; #endif #endif return newPtr; } #ifdef WOLFSSL_TRACK_MEMORY int InitMemoryTracker(void) { if (pthread_mutex_lock(&g_memStatLock) == 0) { g_memStats.totalAllocs = 0; g_memStats.totalDeallocs= 0; g_memStats.totalBytes = 0; g_memStats.peakBytes = 0; g_memStats.currentBytes = 0; XMEMSET(&g_memList, 0, sizeof(g_memList)); pthread_mutex_unlock(&g_memStatLock); } return 0; } void ShowMemoryTracker(void) { if (pthread_mutex_lock(&g_memStatLock) == 0) { printf("total Allocs = %9ld\n", g_memStats.totalAllocs); printf("total Deallocs = %9ld\n", g_memStats.totalDeallocs); printf("total Bytes = %9ld\n", g_memStats.totalBytes); printf("peak Bytes = %9ld\n", g_memStats.peakBytes); printf("current Bytes = %9ld\n", g_memStats.currentBytes); if (g_memList.count > 0) { /* print list of allocations */ qaeMemHeader* header; for (header = g_memList.head; header != NULL; header = header->next) { printf("Leak: Ptr %p, Size %u, Type %d, Heap %p" #ifdef WOLFSSL_DEBUG_MEMORY ", Func %s, Line %d" #endif "\n", (byte*)header + sizeof(qaeMemHeader), (unsigned int)header->size, header->type, header->heap #ifdef WOLFSSL_DEBUG_MEMORY , header->func, header->line #endif ); } } pthread_mutex_unlock(&g_memStatLock); /* cleanup lock */ pthread_mutex_destroy(&g_memStatLock); } } #endif /* WOLFSSL_TRACK_MEMORY */ /************************************** * Memory functions *************************************/ #ifndef QAT_V2 CpaStatus qaeMemInit(void) { if (g_qaeMemFd < 0) { #ifndef QAT_V2 g_qaeMemFd = wc_open_cloexec(QAE_MEM, O_RDWR); if (g_qaeMemFd < 0) { printf("unable to open %s %d\n", QAE_MEM, g_qaeMemFd); return CPA_STATUS_FAIL; } #endif } return CPA_STATUS_SUCCESS; } void qaeMemDestroy(void) { close(g_qaeMemFd); g_qaeMemFd = -1; } #ifdef USE_QAE_STATIC_MEM static CpaStatus userMemListAdd(qae_dev_mem_info_t *pMemInfo) { qae_dev_mem_info_ex_t* pMemInfoEx = (qae_dev_mem_info_ex_t*)pMemInfo->virt_addr; if (g_userMemListCount >= QAE_USER_MEM_MAX_COUNT) { return MEM_INVALID_IDX; } g_pUserMemList[g_userMemListCount] = pMemInfoEx; g_avail_size[g_userMemListCount] = pMemInfoEx->mem_info.available_size; g_lastIndexBySize = g_userMemListCount; g_userMemListCount++; return CPA_STATUS_SUCCESS; } static void userMemListFree(qae_dev_mem_info_t *pMemInfo, int memIdx) { if (memIdx < 0 || memIdx >= g_userMemListCount || g_userMemListCount >= QAE_USER_MEM_MAX_COUNT) { return; } if (memIdx < g_userMemListCount - 1) { /* Replace the deleted index with the last one */ g_pUserMemList[memIdx] = g_pUserMemList[g_userMemListCount - 1]; g_avail_size[memIdx] = g_avail_size[g_userMemListCount - 1]; g_pUserMemList[memIdx]->index = memIdx; } g_userMemListCount--; (void)pMemInfo; } static qae_dev_mem_info_t* userMemLookupBySize(Cpa32U size, int* pMemIdx) { int memIdx; int count = g_userMemListCount; int lastIndex = g_lastIndexBySize; uint16_t *available_size = g_avail_size; for (memIdx = lastIndex; memIdx < count; memIdx++) { if (available_size[memIdx] >= size) { g_lastIndexBySize = memIdx; if (pMemIdx) *pMemIdx = memIdx; return (qae_dev_mem_info_t *)g_pUserMemList[memIdx]; } } for (memIdx = 0; memIdx < lastIndex && memIdx < count; memIdx++) { if (available_size[memIdx] >= size) { g_lastIndexBySize = memIdx; if (pMemIdx) *pMemIdx = memIdx; return (qae_dev_mem_info_t *)g_pUserMemList[memIdx]; } } return NULL; } static qae_dev_mem_info_t* userMemLookupByVirtAddr(void* virt_addr, uint32_t page_offset, int* pMemIdx) { qae_dev_mem_info_ex_t *pMemInfoEx = NULL; void *pageVirtAddr; int memIdx; /* Find the base page virtual address */ pageVirtAddr = (void *)(((QAE_UINT)virt_addr & SYSTEM_PAGE_MASK) - (page_offset << SYSTEM_PAGE_SHIFT)); pMemInfoEx = (qae_dev_mem_info_ex_t*)pageVirtAddr; /* Find the index in g_pUserMemList stored directly in * qae_dev_mem_info_ex_t */ memIdx = pMemInfoEx->index; if (memIdx < 0 || memIdx >= g_userMemListCount) { printf("userMemIndex out of bounds: %d\n", memIdx); return NULL; } if (g_pUserMemList[memIdx] != pMemInfoEx) { printf("userMemIndex virtual address mismatch (memIdx = %d, %p)\n", memIdx, pageVirtAddr); return NULL; } if (pMemIdx) *pMemIdx = memIdx; return (qae_dev_mem_info_t*)pMemInfoEx; } #else static CpaStatus userMemListAdd(qae_dev_mem_info_t *pMemInfo) { if (g_pUserMemList == NULL) { g_pUserMemList = pMemInfo; pMemInfo->pNext = NULL; pMemInfo->pPrev = NULL; g_pUserMemListHead = g_pUserMemList; } else { pMemInfo->pPrev = g_pUserMemList; g_pUserMemList->pNext = pMemInfo; pMemInfo->pNext = NULL; g_pUserMemList = pMemInfo; } return CPA_STATUS_SUCCESS; } static void userMemListFree(qae_dev_mem_info_t *pMemInfo) { qae_dev_mem_info_t *pCurr = NULL; for (pCurr = g_pUserMemListHead; pCurr != NULL; pCurr = pCurr->pNext) { if (pCurr == pMemInfo) { /* If the previous pointer is not NULL */ if (pCurr->pPrev != NULL) { pCurr->pPrev->pNext = pCurr->pNext; if (pCurr->pNext) { pCurr->pNext->pPrev = pCurr->pPrev; } else { g_pUserMemList = pCurr->pPrev; } } else if (pCurr->pNext != NULL) { pCurr->pNext->pPrev = NULL; g_pUserMemListHead = pCurr->pNext; } else { g_pUserMemList = NULL; g_pUserMemListHead = NULL; } break; } } } static qae_dev_mem_info_t* userMemLookupBySize(Cpa32U size) { qae_dev_mem_info_t *pCurr = NULL; for (pCurr = g_pUserMemListHead; pCurr != NULL; pCurr = pCurr->pNext) { if (pCurr->available_size >= size) { return pCurr; } } return NULL; } static qae_dev_mem_info_t* userMemLookupByVirtAddr(void* virt_addr, uint32_t page_offset) { qae_dev_mem_info_t *pCurr = NULL; for (pCurr = g_pUserMemListHead; pCurr != NULL; pCurr = pCurr->pNext) { if ((QAE_UINT)pCurr->virt_addr <= (QAE_UINT)virt_addr && ((QAE_UINT)pCurr->virt_addr + pCurr->size) > (QAE_UINT)virt_addr) { return pCurr; } } (void)page_offset; return NULL; } #endif static void* qaeMemAllocNUMA(Cpa32U size, Cpa32U node, Cpa32U alignment, word16* p_page_offset) { int ret = 0; qae_dev_mem_info_t* pMemInfo = NULL; void* pVirtAddress = NULL; void* pOriginalAddress = NULL; QAE_UINT padding = 0; QAE_UINT aligned_address = 0; const uint64_t magic = QAEM_MAGIC_NUM; #ifdef USE_QAE_STATIC_MEM int memIdx; qae_dev_mem_info_t memInfo; qae_dev_mem_info_ex_t* pMemInfoEx; #endif if (size == 0 || alignment == 0) { printf("Invalid size or alignment parameter\n"); return NULL; } if (g_qaeMemFd < 0) { qaeMemInit(); } if ( (pMemInfo = userMemLookupBySize(size + alignment #ifdef USE_QAE_STATIC_MEM , &memIdx #endif )) != NULL) { /* calculate address */ pOriginalAddress = (void*)((QAE_UINT)pMemInfo->virt_addr + (QAE_UINT)(pMemInfo->size - pMemInfo->available_size)); /* calculate aligned address */ padding = (QAE_UINT)pOriginalAddress % alignment; aligned_address = ((QAE_UINT)pOriginalAddress) - padding + alignment; /* reduce available size */ pMemInfo->available_size -= (size + (aligned_address - (QAE_UINT)pOriginalAddress)); pMemInfo->allocations += 1; #ifdef USE_QAE_STATIC_MEM /* cache index's available size */ g_avail_size[memIdx] = pMemInfo->available_size; #endif *p_page_offset = (word16)( (QAE_UINT)aligned_address >> SYSTEM_PAGE_SHIFT) - ((QAE_UINT)pMemInfo->virt_addr >> SYSTEM_PAGE_SHIFT); return (void*)aligned_address; } #ifdef USE_QAE_STATIC_MEM pMemInfo = &memInfo; #else pMemInfo = malloc(sizeof(qae_dev_mem_info_t)); if (pMemInfo == NULL) { printf("unable to allocate pMemInfo buffer\n"); return NULL; } #endif pMemInfo->allocations = 0; pMemInfo->size = USER_MEM_OFFSET + size; pMemInfo->size = pMemInfo->size % PAGE_SIZE ? ((pMemInfo->size / PAGE_SIZE) + 1) * PAGE_SIZE : pMemInfo->size; #ifdef SAL_IOMMU_CODE pMemInfo->size = icp_sal_iommu_get_remap_size(pMemInfo->size); #endif pMemInfo->nodeId = node; ret = ioctl(g_qaeMemFd, DEV_MEM_IOC_MEMALLOC, pMemInfo); if (ret != 0) { printf("ioctl call failed: ret %d, errno %d (%s)\n", ret, errno, strerror(errno)); return NULL; } pMemInfo->virt_addr = mmap((caddr_t)0, pMemInfo->size, PROT_READ|PROT_WRITE, MAP_SHARED, g_qaeMemFd, (pMemInfo->id * getpagesize())); if (pMemInfo->virt_addr == (caddr_t)MAP_FAILED) { printf("mmap failed\n"); ret = ioctl(g_qaeMemFd, DEV_MEM_IOC_MEMFREE, pMemInfo); if (ret != 0) { printf("ioctl call failed: ret %d, errno %d (%s)\n", ret, errno, strerror(errno)); } #ifndef USE_QAE_STATIC_MEM free(pMemInfo); #endif return NULL; } pMemInfo->available_size = pMemInfo->size - size - USER_MEM_OFFSET; pMemInfo->allocations = 1; memcpy(pMemInfo->virt_addr, pMemInfo, sizeof(qae_dev_mem_info_t)); #ifdef USE_QAE_STATIC_MEM pMemInfoEx = (qae_dev_mem_info_ex_t *)pMemInfo->virt_addr; pMemInfoEx->index = g_userMemListCount; #endif memcpy(pMemInfo->virt_addr, &magic, sizeof(uint64_t)); pVirtAddress = (void *)((QAE_UINT)pMemInfo->virt_addr + USER_MEM_OFFSET); if (userMemListAdd(pMemInfo) != CPA_STATUS_SUCCESS) { printf("Error on mem list add\n"); #ifndef USE_QAE_STATIC_MEM free(pMemInfo); #endif return NULL; } *p_page_offset = 0; return pVirtAddress; } static void qaeMemFreeNUMA(void** ptr, word16 page_offset) { int ret = 0; qae_dev_mem_info_t *pMemInfo = NULL; void* pVirtAddress = NULL; #ifdef USE_QAE_STATIC_MEM qae_dev_mem_info_t memInfo; /* temp buffer */ int memIdx; #endif if (ptr == NULL) return; pVirtAddress = *ptr; if (pVirtAddress == NULL) { printf("qaeMemFreeNUMA: Invalid virtual address\n"); return; } if ((pMemInfo = userMemLookupByVirtAddr(pVirtAddress, page_offset #ifdef USE_QAE_STATIC_MEM , &memIdx #endif )) != NULL) { pMemInfo->allocations -= 1; #ifdef USE_QAE_STATIC_MEM if (memIdx < QAE_USER_MEM_MAX_COUNT && pMemInfo->allocations == 0) { pMemInfo->available_size = pMemInfo->size - USER_MEM_OFFSET; g_avail_size[memIdx] = pMemInfo->available_size; } #endif if (pMemInfo->allocations != 0 #ifdef USE_QAE_STATIC_MEM || memIdx < QAE_USER_MEM_MAX_COUNT #endif ) { *ptr = NULL; return; } } else { printf("userMemLookupByVirtAddr failed\n"); return; } #ifdef USE_QAE_STATIC_MEM /* use a temp copy of memory info */ memInfo = *pMemInfo; userMemListFree(pMemInfo->virt_addr, memIdx); pMemInfo = &memInfo; #endif ret = munmap(pMemInfo->virt_addr, pMemInfo->size); if (ret != 0) { printf("munmap failed, ret = %d\n",ret); } ret = ioctl(g_qaeMemFd, DEV_MEM_IOC_MEMFREE, pMemInfo); if (ret != 0) { printf("ioctl call failed, ret = %d\n",ret); } #ifndef USE_QAE_STATIC_MEM userMemListFree(pMemInfo); free(pMemInfo); #endif *ptr = NULL; return; } QAE_PHYS_ADDR qaeVirtToPhysNUMA(void* pVirtAddress) { qae_dev_mem_info_t *pMemInfo = NULL; void *pVirtPageAddress = NULL; QAE_UINT offset = 0; uint64_t *magic; if (pVirtAddress == NULL) { printf("qaeVirtToPhysNUMA: Null virtual address pointer\n"); return (QAE_PHYS_ADDR)0; } pVirtPageAddress = ((int *)((( (QAE_UINT)pVirtAddress)) & (SYSTEM_PAGE_MASK))); offset = (QAE_UINT)pVirtAddress - (QAE_UINT)pVirtPageAddress; do { pMemInfo = (qae_dev_mem_info_t *)pVirtPageAddress; magic = (uint64_t *)pMemInfo; if ((QAEM_MAGIC_NUM == *magic) && (pMemInfo->virt_addr == pVirtPageAddress)) { break; } pVirtPageAddress = (void*)( (QAE_UINT)pVirtPageAddress - SYSTEM_PAGE_SIZE); offset += SYSTEM_PAGE_SIZE; } while (pMemInfo->virt_addr != pVirtPageAddress); return (QAE_PHYS_ADDR)(pMemInfo->phy_addr + offset); } #endif /* !QAT_V2 */ #endif /* HAVE_INTEL_QA */