llmnpc - llama.cpp/ggml/src/ggml-hexagon/htp/act-ops.c

  1#pragma clang diagnostic ignored "-Wunused-variable"
  2#pragma clang diagnostic ignored "-Wunused-function"
  3#pragma clang diagnostic ignored "-Wunused-but-set-variable"
  4
  5#include <HAP_farf.h>
  6#include <HAP_perf.h>
  7
  8#include <math.h>
  9#include <string.h>
 10
 11#include "hex-dma.h"
 12#include "hvx-utils.h"
 13
 14#define GGML_COMMON_DECL_C
 15#include "ggml-common.h"
 16#include "htp-ctx.h"
 17#include "htp-msg.h"
 18#include "htp-ops.h"
 19
 20#define htp_act_preamble3              \
 21    const uint32_t ne00 = src0->ne[0]; \
 22    const uint32_t ne01 = src0->ne[1]; \
 23    const uint32_t ne02 = src0->ne[2]; \
 24    const uint32_t ne03 = src0->ne[3]; \
 25                                       \
 26    const uint32_t ne10 = src1->ne[0]; \
 27    const uint32_t ne11 = src1->ne[1]; \
 28    const uint32_t ne12 = src1->ne[2]; \
 29    const uint32_t ne13 = src1->ne[3]; \
 30                                       \
 31    const uint32_t ne0 = dst->ne[0];   \
 32    const uint32_t ne1 = dst->ne[1];   \
 33    const uint32_t ne2 = dst->ne[2];   \
 34    const uint32_t ne3 = dst->ne[3];   \
 35                                       \
 36    const uint32_t nb00 = src0->nb[0]; \
 37    const uint32_t nb01 = src0->nb[1]; \
 38    const uint32_t nb02 = src0->nb[2]; \
 39    const uint32_t nb03 = src0->nb[3]; \
 40                                       \
 41    const uint32_t nb10 = src1->nb[0]; \
 42    const uint32_t nb11 = src1->nb[1]; \
 43    const uint32_t nb12 = src1->nb[2]; \
 44    const uint32_t nb13 = src1->nb[3]; \
 45                                       \
 46    const uint32_t nb0 = dst->nb[0];   \
 47    const uint32_t nb1 = dst->nb[1];   \
 48    const uint32_t nb2 = dst->nb[2];   \
 49    const uint32_t nb3 = dst->nb[3];
 50
 51#define htp_act_preamble2              \
 52    const uint32_t ne00 = src0->ne[0]; \
 53    const uint32_t ne01 = src0->ne[1]; \
 54    const uint32_t ne02 = src0->ne[2]; \
 55    const uint32_t ne03 = src0->ne[3]; \
 56                                       \
 57    const uint32_t ne0 = dst->ne[0];   \
 58    const uint32_t ne1 = dst->ne[1];   \
 59    const uint32_t ne2 = dst->ne[2];   \
 60    const uint32_t ne3 = dst->ne[3];   \
 61                                       \
 62    const uint32_t nb00 = src0->nb[0]; \
 63    const uint32_t nb01 = src0->nb[1]; \
 64    const uint32_t nb02 = src0->nb[2]; \
 65    const uint32_t nb03 = src0->nb[3]; \
 66                                       \
 67    const uint32_t nb0 = dst->nb[0];   \
 68    const uint32_t nb1 = dst->nb[1];   \
 69    const uint32_t nb2 = dst->nb[2];   \
 70    const uint32_t nb3 = dst->nb[3];
 71
 72static void glu_swiglu_f32_per_thread(const struct htp_tensor * src0,
 73                                       const struct htp_tensor * src1,
 74                                       struct htp_tensor *       dst,
 75                                       const int32_t *           op_params,
 76                                       struct htp_spad *         src0_spad,
 77                                       struct htp_spad *         src1_spad,
 78                                       struct htp_spad *         dst_spad,
 79                                       uint32_t                  nth,
 80                                       uint32_t                  ith,
 81                                       uint32_t                  src0_nrows_per_thread,
 82                                       dma_queue *               dma_queue) {
 83    htp_act_preamble3;
 84
 85    size_t src0_row_size = nb01;
 86    size_t src1_row_size = nb11;
 87    size_t dst_row_size  = nb1;
 88
 89
 90
 91    const uint32_t src0_nrows = ne01 * ne02 * ne03;  // src0 rows
 92
 93    const uint32_t src0_start_row = src0_nrows_per_thread * ith;
 94    const uint32_t src0_end_row   = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);
 95
 96    // no work for this thread
 97    if (src0_start_row >= src0_end_row) {
 98        return;
 99    }
100
101    uint64_t t1, t2;
102    t1 = HAP_perf_get_qtimer_count();
103
104    const uint8_t * restrict data_src0 = (const uint8_t *) src0->data;
105    const uint8_t * restrict data_src1 = (const uint8_t *) src1->data;
106    uint8_t * restrict data_dst        = (uint8_t *) dst->data;
107
108    const bool src1_valid = src1->ne[0];
109    const int  nc         = (src1_valid) ? ne00 : ne00 / 2;
110    if (!src1_valid) {
111        const int32_t swapped = op_params[1];
112        data_src1             = data_src0;
113        src1_row_size         = src0_row_size;
114
115        const size_t nc_in_bytes = nc * SIZEOF_FP32;
116        data_src0 += swapped ? nc_in_bytes : 0;
117        data_src1 += swapped ? 0 : nc_in_bytes;
118    }
119
120    const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);
121    const size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN);
122    const size_t dst_row_size_aligned  = hex_round_up(dst_row_size, VLEN);
123
124    uint8_t * restrict src0_spad_data = src0_spad->data + (ith * src0_spad->size_per_thread);
125    uint8_t * restrict src1_spad_data = src1_spad->data + (ith * src1_spad->size_per_thread);
126    uint8_t * restrict dst_spad_data  = dst_spad->data + (ith * dst_spad->size_per_thread);
127
128    // While given src0_spad->size_per_thread, divide it to two ping-pong buffer for src0
129    size_t src0_spad_half_size = src0_spad->size_per_thread / 2;
130    size_t src1_spad_half_size = src1_spad->size_per_thread / 2;
131    size_t dst_spad_half_size  = dst_spad->size_per_thread / 2;
132
133    const int BLOCK = src0_spad_half_size / src0_row_size_aligned;  // How many rows can we process in one block
134    if (BLOCK == 0) {
135        FARF(ERROR,
136             "swiglu-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n",
137             src0_spad->size_per_thread, src0_row_size_aligned);
138        return;
139    }
140
141    // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379
142    for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) {
143        const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);
144
145        // Dummy DMA transation for sequencing (interleaving dst,src,dst,...)
146        dma_queue_push_vtcm_to_ddr(dma_queue,
147            dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)),
148            dst_row_size, dst_row_size_aligned, 0);
149
150        dma_queue_push_ddr_to_vtcm(dma_queue,
151            dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)),
152            src0_row_size_aligned, src0_row_size, block_size);
153        dma_queue_push_ddr_to_vtcm(dma_queue,
154            dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_size)),
155            src1_row_size_aligned, src1_row_size, block_size);
156    }
157
158    for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) {
159        const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);
160
161        float * dst_spad  = (float *) dma_queue_pop(dma_queue).src;
162        float * src0_spad = (float *) dma_queue_pop(dma_queue).dst;
163        float * src1_spad = (float *) dma_queue_pop(dma_queue).dst;
164
165        for (uint32_t ib = 0; ib < block_size; ib++) {
166            const float * src0_spad_ptr = src0_spad + ib * (src0_row_size_aligned / sizeof(float));
167            const float * src1_spad_ptr = src1_spad + ib * (src1_row_size_aligned / sizeof(float));
168            float *       dst_spad_ptr  = dst_spad + ib * (dst_row_size_aligned / sizeof(float));
169
170            //swiglu(x) = x1 * sigmoid(x0)
171            hvx_sigmoid_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, nc);
172            hvx_mul_mul_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (const uint8_t *) dst_spad_ptr,
173                                (const uint8_t *) src1_spad_ptr, nc);
174        }
175
176        dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), dst_row_size,
177                                   dst_row_size_aligned, block_size);
178
179        // prefetch N+2 loop iteration if any
180        const uint32_t pref_block = (ir + BLOCK * 2);
181        if (pref_block < src0_end_row) {
182            const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block);
183            dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)),
184                                       src0_row_size_aligned, src0_row_size, pref_block_size);
185            dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_size)),
186                                       src1_row_size_aligned, src1_row_size, pref_block_size);
187        }
188    }
189
190    dma_queue_flush(dma_queue);
191
192    t2 = HAP_perf_get_qtimer_count();
193
194    FARF(HIGH, "swiglu-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth,
195         ne00, ne01, ne02, ne03, src0_start_row, src0_end_row, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3,
196         (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
197}
198
199static void glu_swiglu_oai_f32_per_thread(const struct htp_tensor * src0,
200                                           const struct htp_tensor * src1,
201                                           struct htp_tensor *       dst,
202                                           const int32_t *           op_params,
203                                           struct htp_spad *         src0_spad,
204                                           struct htp_spad *         src1_spad,
205                                           struct htp_spad *         dst_spad,
206                                           uint32_t                  nth,
207                                           uint32_t                  ith,
208                                           uint32_t                  src0_nrows_per_thread,
209                                           dma_queue *               dma_queue) {
210    htp_act_preamble3;
211
212    uint64_t t1, t2;
213    t1 = HAP_perf_get_qtimer_count();
214
215    size_t src0_row_size = nb01;
216    size_t src1_row_size = nb11;
217    size_t dst_row_size  = nb1;
218
219    const uint32_t src0_nrows = ne01 * ne02 * ne03;  // src0 rows
220
221    const uint32_t src0_start_row = src0_nrows_per_thread * ith;
222    const uint32_t src0_end_row   = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);
223
224    // no work for this thread
225    if (src0_start_row >= src0_end_row) {
226        return;
227    }
228
229    const uint8_t * restrict data_src0 = (const uint8_t *) src0->data;
230    const uint8_t * restrict data_src1 = (const uint8_t *) src1->data;
231    uint8_t * restrict data_dst        = (uint8_t *) dst->data;
232
233    const bool src1_valid = src1->ne[0];
234    const int  nc         = (src1_valid) ? ne00 : ne00 / 2;
235    if (!src1_valid) {
236        const int32_t swapped = op_params[1];
237        data_src1             = data_src0;
238        src1_row_size         = src0_row_size;
239
240        const size_t nc_in_bytes = nc * SIZEOF_FP32;
241        data_src0 += swapped ? nc_in_bytes : 0;
242        data_src1 += swapped ? 0 : nc_in_bytes;
243    }
244
245    const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);
246    const size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN);
247    const size_t dst_row_size_aligned  = hex_round_up(dst_row_size, VLEN);
248
249    uint8_t * restrict src0_spad_data = src0_spad->data + (ith * src0_spad->size_per_thread);
250    uint8_t * restrict src1_spad_data = src1_spad->data + (ith * src1_spad->size_per_thread);
251    uint8_t * restrict dst_spad_data  = dst_spad->data + (ith * dst_spad->size_per_thread);
252
253    // While given src0_spad->size_per_thread, divide it to two ping-pong buffer for src0
254    size_t src0_spad_half_size = src0_spad->size_per_thread / 2;
255    size_t src1_spad_half_size = src1_spad->size_per_thread / 2;
256    size_t dst_spad_half_size  = dst_spad->size_per_thread / 2;
257
258    const int BLOCK = src0_spad_half_size / src0_row_size_aligned;  // How many rows can we process in one block
259    if (BLOCK == 0) {
260        FARF(ERROR,
261             "swiglu-oai-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least "
262             "%zu\n",
263             src0_spad->size_per_thread, src0_row_size_aligned);
264        return;
265    }
266    const float alpha = ((const float *) (op_params))[2];
267    const float limit = ((const float *) (op_params))[3];
268
269    // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379
270    for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) {
271        const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);
272
273        // Dummy DMA transation for sequencing (interleaving dst,src,dst,...)
274        dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)),
275                                   dst_row_size, dst_row_size_aligned, 0);
276
277        dma_queue_push_ddr_to_vtcm(
278            dma_queue,
279            dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)),
280            src0_row_size_aligned, src0_row_size, block_size);
281        dma_queue_push_ddr_to_vtcm(
282            dma_queue,
283            dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_size)),
284            src1_row_size_aligned, src1_row_size, block_size);
285    }
286
287    for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) {
288        const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);
289
290        float * dst_spad  = (float *) dma_queue_pop(dma_queue).src;
291        float * src0_spad = (float *) dma_queue_pop(dma_queue).dst;
292        float * src1_spad = (float *) dma_queue_pop(dma_queue).dst;
293
294        for (uint32_t ib = 0; ib < block_size; ib++) {
295            const float * src0_spad_ptr = src0_spad + ib * (src0_row_size_aligned / sizeof(float));
296            const float * src1_spad_ptr = src1_spad + ib * (src1_row_size_aligned / sizeof(float));
297            float *       dst_spad_ptr  = dst_spad + ib * (dst_row_size_aligned / sizeof(float));
298
299            // x (src0_spad_data) = std::min(src0_p[k], limit);
300            hvx_min_scalar_f32((uint8_t *) src0_spad_ptr, (const uint8_t *) src0_spad_ptr, limit, nc);
301            // y1 (src1_spad_data) = std::clamp(src1_p[k], -limit, limit);
302            hvx_clamp_scalar_f32((uint8_t *) src1_spad_ptr, (const uint8_t *) src1_spad_ptr, -limit, limit, nc);
303            // y (src1_spad_data)  = y1 + 1.f
304            hvx_add_scalar_f32((uint8_t *) src1_spad_ptr, (const uint8_t *) src1_spad_ptr, 1.0, nc);
305            // x1 (dst_spad_data) = alpha * (x)
306            hvx_mul_scalar_f32((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, alpha, nc);
307            // x2 (dst_spad_data) = sigmoid(x1) = 1/(1+exp(-x1))
308            hvx_sigmoid_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) dst_spad_ptr, nc);
309            // out = x * sigmoid(alpha * x) * (y + 1.f)
310            hvx_mul_mul_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (const uint8_t *) dst_spad_ptr,
311                                (const uint8_t *) src1_spad_ptr, nc);
312        }
313
314        dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), dst_row_size,
315                                   dst_row_size_aligned, block_size);
316
317        // prefetch N+2 loop iteration if any
318        const uint32_t pref_block = (ir + BLOCK * 2);
319        if (pref_block < src0_end_row) {
320            const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block);
321            dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)),
322                                       src0_row_size_aligned, src0_row_size, pref_block_size);
323            dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_size)),
324                                       src1_row_size_aligned, src1_row_size, pref_block_size);
325        }
326    }
327
328    dma_queue_flush(dma_queue);
329
330    t2 = HAP_perf_get_qtimer_count();
331
332    FARF(HIGH, "swiglu-oai-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth, src0->ne[0],
333         src0->ne[1], src0->ne[2], src0->ne[3], src0_start_row, src0_end_row, src1->ne[0], src1->ne[1], src1->ne[2],
334         src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
335}
336
337
338static void unary_gelu_f32_per_thread(const struct htp_tensor * src0,
339                                       struct htp_tensor *       dst,
340                                       const int32_t *           op_params,
341                                       struct htp_spad *         src0_spad,
342                                       struct htp_spad *         dst_spad,
343                                       uint32_t                  nth,
344                                       uint32_t                  ith,
345                                       uint32_t                  src0_nrows_per_thread,
346                                       dma_queue *               dma_queue) {
347    htp_act_preamble2;
348
349    uint64_t t1, t2;
350    t1 = HAP_perf_get_qtimer_count();
351
352    const size_t src0_row_size = nb01;
353    const size_t dst_row_size  = nb1;
354    const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);
355    const size_t dst_row_size_aligned  = hex_round_up(dst_row_size, VLEN);
356
357    const uint32_t src0_nrows = ne01 * ne02 * ne03;
358
359    const uint32_t src0_start_row = src0_nrows_per_thread * ith;
360    const uint32_t src0_end_row   = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);
361
362    // no work for this thread
363    if (src0_start_row >= src0_end_row) {
364        return;
365    }
366
367    const uint8_t * data_src0 = (const uint8_t *) src0->data;
368    uint8_t * data_dst        = (uint8_t *) dst->data;
369
370    uint8_t * src0_spad_data = src0_spad->data + (ith * src0_spad->size_per_thread);
371    uint8_t * dst_spad_data  = dst_spad->data  + (ith * dst_spad->size_per_thread);
372
373    // While given src0_spad->size_per_thread, divide it to two ping-pong buffer for src0
374    size_t src0_spad_half_size = src0_spad->size_per_thread / 2;
375    size_t dst_spad_half_size  = dst_spad->size_per_thread  / 2;
376
377    // In gelu = x*sigmoid(x*1.702)
378    const int BLOCK = src0_spad_half_size / src0_row_size_aligned; // How many rows can we process in one block
379
380    if (BLOCK == 0) {
381        FARF(ERROR, "gelu-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n",
382                src0_spad->size_per_thread, src0_row_size_aligned);
383        return;
384    }
385
386    // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379
387    for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) {
388        const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);
389
390        // Dummy DMA transation for sequencing (interleaving dst,src,dst,...)
391        dma_queue_push_vtcm_to_ddr(dma_queue,
392            dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)),
393            dst_row_size, dst_row_size_aligned, 0);
394
395        dma_queue_push_ddr_to_vtcm(dma_queue,
396            dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)),
397            src0_row_size_aligned, src0_row_size, block_size);
398    }
399
400    for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) {
401        const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);
402
403        float* dst_spad  = (float *) dma_queue_pop(dma_queue).src;
404        float* src0_spad = (float *) dma_queue_pop(dma_queue).dst;
405
406        for (uint32_t ib = 0; ib < block_size; ib++) {
407            const float* src0_spad_ptr = src0_spad + ib * (src0_row_size_aligned / sizeof(float));
408            float* dst_spad_ptr        = dst_spad  + ib * (dst_row_size_aligned  / sizeof(float));
409
410            // gelu = x * sigmoid(1.702 * x) // current implementation
411            hvx_mul_scalar_f32((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (float) 1.702, ne0);
412            hvx_sigmoid_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) dst_spad_ptr, ne0);
413            hvx_mul_f32_aaa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (const uint8_t *) dst_spad_ptr, ne0);
414        }
415
416        dma_queue_push_vtcm_to_ddr(dma_queue,
417            dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad),
418            dst_row_size, dst_row_size_aligned, block_size);
419
420        // prefetch N+2 loop iteration if any
421        const uint32_t pref_block = (ir + BLOCK * 2);
422        if (pref_block < src0_end_row) {
423            const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block);
424            dma_queue_push_ddr_to_vtcm(dma_queue,
425                dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)),
426                src0_row_size_aligned, src0_row_size, pref_block_size);
427        }
428    }
429
430    dma_queue_flush(dma_queue);
431
432    t2 = HAP_perf_get_qtimer_count();
433
434    FARF(HIGH, "gelu-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", ith, nth, ne00, ne01, ne02,
435         ne03, src0_start_row, src0_end_row, ne0, ne1, ne2, ne3, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
436}
437
438static void unary_gelu_f32(unsigned int n, unsigned int i, void * data) {
439    struct htp_ops_context * octx = (struct htp_ops_context *) data;
440    unary_gelu_f32_per_thread(&octx->src0, &octx->dst, octx->op_params, &octx->src0_spad, &octx->dst_spad, n, i,
441                               octx->src0_nrows_per_thread, octx->ctx->dma[i]);
442}
443
444
445
446static void unary_silu_f32_per_thread(const struct htp_tensor * src0,
447                                       struct htp_tensor *       dst,
448                                       const int32_t *           op_params,
449                                       struct htp_spad *         src0_spad,
450                                       struct htp_spad *         dst_spad,
451                                       uint32_t                  nth,
452                                       uint32_t                  ith,
453                                       uint32_t                  src0_nrows_per_thread,
454                                       dma_queue *               dma_queue) {
455    htp_act_preamble2;
456
457    uint64_t t1, t2;
458    t1 = HAP_perf_get_qtimer_count();
459
460    const size_t src0_row_size = nb01;
461    const size_t dst_row_size  = nb1;
462    const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);
463    const size_t dst_row_size_aligned  = hex_round_up(dst_row_size, VLEN);
464
465    const uint32_t src0_nrows = ne01 * ne02 * ne03;
466
467    const uint32_t src0_start_row = src0_nrows_per_thread * ith;
468    const uint32_t src0_end_row   = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);
469
470    // no work for this thread
471    if (src0_start_row >= src0_end_row) {
472        return;
473    }
474
475    const uint8_t * data_src0 = (const uint8_t *) src0->data;
476    uint8_t * data_dst        = (uint8_t *) dst->data;
477
478    uint8_t * src0_spad_data = src0_spad->data + (ith * src0_spad->size_per_thread);
479    uint8_t * dst_spad_data  = dst_spad->data  + (ith * dst_spad->size_per_thread);
480
481    // While given src0_spad->size_per_thread, divide it to two ping-pong buffer for src0
482    size_t src0_spad_half_size = src0_spad->size_per_thread / 2;
483    size_t dst_spad_half_size  = dst_spad->size_per_thread  / 2;
484
485    const int BLOCK = src0_spad_half_size / src0_row_size_aligned; // How many rows can we process in one block
486
487    if (BLOCK == 0) {
488        FARF(ERROR, "silu-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n",
489                src0_spad->size_per_thread, src0_row_size_aligned);
490        return;
491    }
492
493    // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379
494    for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) {
495        const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);
496
497        // Dummy DMA transation for sequencing (interleaving dst,src,dst,...)
498        dma_queue_push_vtcm_to_ddr(dma_queue,
499            dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)),
500            dst_row_size, dst_row_size_aligned, 0);
501
502        dma_queue_push_ddr_to_vtcm(dma_queue,
503            dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)),
504            src0_row_size_aligned, src0_row_size, block_size);
505    }
506
507    for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) {
508        const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);
509
510        float* dst_spad  = (float *) dma_queue_pop(dma_queue).src;
511        float* src0_spad = (float *) dma_queue_pop(dma_queue).dst;
512
513        for (uint32_t ib = 0; ib < block_size; ib++) {
514            const float* src0_spad_ptr = src0_spad + ib * (src0_row_size_aligned / sizeof(float));
515            float* dst_spad_ptr        = dst_spad  + ib * (dst_row_size_aligned  / sizeof(float));
516
517            // silu = x * sigmoid(x)
518            hvx_sigmoid_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, ne0);
519            hvx_mul_f32_aaa((uint8_t *) dst_spad_ptr, (const uint8_t *) src0_spad_ptr, (const uint8_t *) dst_spad_ptr, ne0);
520        }
521
522        dma_queue_push_vtcm_to_ddr(dma_queue,
523            dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad),
524            dst_row_size, dst_row_size_aligned, block_size);
525
526        // prefetch N+2 loop iteration if any
527        const uint32_t pref_block = (ir + BLOCK * 2);
528        if (pref_block < src0_end_row) {
529            const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block);
530            dma_queue_push_ddr_to_vtcm(dma_queue,
531                dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)),
532                src0_row_size_aligned, src0_row_size, pref_block_size);
533        }
534    }
535
536    dma_queue_flush(dma_queue);
537
538    t2 = HAP_perf_get_qtimer_count();
539
540    FARF(HIGH, "silu-f32 %d/%d: %ux%ux%ux%u (%u:%u) -> %ux%ux%ux%u usec %u\n", ith, nth, ne00, ne01, ne02,
541         ne03, src0_start_row, src0_end_row, ne0, ne1, ne2, ne3, (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
542}
543
544static const float GELU_COEF_A     = 0.044715f;
545static const float SQRT_2_OVER_PI  = 0.79788456080286535587989211986876f;
546
547static void glu_geglu_f32_per_thread(const struct htp_tensor * src0,
548                                       const struct htp_tensor * src1,
549                                       struct htp_tensor *       dst,
550                                       const int32_t *           op_params,
551                                       struct htp_spad *         src0_spad,
552                                       struct htp_spad *         src1_spad,
553                                       struct htp_spad *         dst_spad,
554                                       uint32_t                  nth,
555                                       uint32_t                  ith,
556                                       uint32_t                  src0_nrows_per_thread,
557                                       dma_queue *               dma_queue) {
558    htp_act_preamble3;
559
560    size_t src0_row_size = nb01;
561    size_t src1_row_size = nb11;
562    size_t dst_row_size  = nb1;
563
564    uint64_t t1, t2;
565    t1 = HAP_perf_get_qtimer_count();
566
567    const uint32_t src0_nrows = ne01 * ne02 * ne03;  // src0 rows
568
569    const uint32_t src0_start_row = src0_nrows_per_thread * ith;
570    const uint32_t src0_end_row   = MIN(src0_start_row + src0_nrows_per_thread, src0_nrows);
571
572    // no work for this thread
573    if (src0_start_row >= src0_end_row) {
574        return;
575    }
576
577    const uint8_t * restrict data_src0 = (const uint8_t *) src0->data;
578    const uint8_t * restrict data_src1 = (const uint8_t *) src1->data;
579    uint8_t * restrict data_dst        = (uint8_t *) dst->data;
580
581    const bool src1_valid = src1->ne[0];
582    const int  nc         = (src1_valid) ? ne00 : ne00 / 2;
583    if (!src1_valid) {
584        const int32_t swapped = op_params[1];
585        data_src1             = data_src0;
586        src1_row_size         = src0_row_size;
587
588        const size_t nc_in_bytes = nc * SIZEOF_FP32;
589        data_src0 += swapped ? nc_in_bytes : 0;
590        data_src1 += swapped ? 0 : nc_in_bytes;
591    }
592
593    const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);
594    const size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN);
595    const size_t dst_row_size_aligned  = hex_round_up(dst_row_size, VLEN);
596
597    uint8_t * restrict src0_spad_data = src0_spad->data + (ith * src0_spad->size_per_thread);
598    uint8_t * restrict src1_spad_data = src1_spad->data + (ith * src1_spad->size_per_thread);
599    uint8_t * restrict dst_spad_data  = dst_spad->data + (ith * dst_spad->size_per_thread);
600
601    // While given src0_spad->size_per_thread, divide it to two ping-pong buffer for src0
602    size_t src0_spad_half_size = src0_spad->size_per_thread / 2;
603    size_t src1_spad_half_size = src1_spad->size_per_thread / 2;
604    size_t dst_spad_half_size  = dst_spad->size_per_thread / 2;
605
606    const int BLOCK = src0_spad_half_size / src0_row_size_aligned;  // How many rows can we process in one block
607    if (BLOCK == 0) {
608        FARF(ERROR,
609             "geglu-f32 : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n",
610             src0_spad->size_per_thread, src0_row_size_aligned);
611        return;
612    }
613
614    // See discussion: https://github.com/ggml-org/llama.cpp/pull/18151#issuecomment-3678235379
615    for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) {
616        const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);
617
618        // Dummy DMA transation for sequencing (interleaving dst,src,dst,...)
619        dma_queue_push_vtcm_to_ddr(dma_queue,
620            dma_make_ptr(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)),
621            dst_row_size, dst_row_size_aligned, 0);
622
623        dma_queue_push_ddr_to_vtcm(dma_queue,
624            dma_make_ptr(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_size)),
625            src0_row_size_aligned, src0_row_size, block_size);
626        dma_queue_push_ddr_to_vtcm(dma_queue,
627            dma_make_ptr(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_size)),
628            src1_row_size_aligned, src1_row_size, block_size);
629    }
630
631    for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) {
632        const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);
633
634        float * dst_spad  = (float *) dma_queue_pop(dma_queue).src;
635        float * src0_spad = (float *) dma_queue_pop(dma_queue).dst;
636        float * src1_spad = (float *) dma_queue_pop(dma_queue).dst;
637
638        for (uint32_t ib = 0; ib < block_size; ib++) {
639            const uint8_t * src0_spad_ptr = (const uint8_t *)(src0_spad + ib * (src0_row_size_aligned / sizeof(float)));
640            const uint8_t * src1_spad_ptr = (const uint8_t *)(src1_spad + ib * (src1_row_size_aligned / sizeof(float)));
641            uint8_t *       dst_spad_ptr  = (uint8_t *)(dst_spad + ib * (dst_row_size_aligned / sizeof(float)));
642
643            // geglu tanh implementation
644            // geglu(x, g) = gelu(x) * g
645            // gelu(x) = 0.5f*x*(1.0f + tanhf(SQRT_2_OVER_PI*x*(1.0f + GELU_COEF_A*x*x)))
646            hvx_mul_f32_aaa(dst_spad_ptr, src0_spad_ptr, src0_spad_ptr, nc);                       // res = x*x
647            hvx_mul_scalar_f32_aa(dst_spad_ptr, (const uint8_t *)dst_spad_ptr, GELU_COEF_A, nc);   // res = res * GELU_COEF_A
648            hvx_add_scalar_f32_aa(dst_spad_ptr, (const uint8_t *)dst_spad_ptr, 1.0f, nc);          // res = res + 1.0f
649            hvx_mul_f32_aaa(dst_spad_ptr, src0_spad_ptr, (const uint8_t *)dst_spad_ptr, nc);       // res = res * x
650            hvx_mul_scalar_f32_aa(dst_spad_ptr, (const uint8_t*)dst_spad_ptr, SQRT_2_OVER_PI, nc); // res = result * SQRT_2_OVER_PI
651            hvx_tanh_f32_aa((uint8_t *) dst_spad_ptr, (const uint8_t *) dst_spad_ptr, nc);         // res = tanh(res)
652            hvx_add_scalar_f32_aa(dst_spad_ptr, (const uint8_t*)dst_spad_ptr, 1.0f, nc);           // res = res + 1.0f
653            hvx_mul_f32_aaa(dst_spad_ptr, src0_spad_ptr, (const uint8_t *)dst_spad_ptr, nc);       // res = res * x
654            hvx_mul_scalar_f32_aa(dst_spad_ptr, (const uint8_t *)dst_spad_ptr, 0.5f, nc);          // res = res + 0.5f
655            hvx_mul_f32_aaa(dst_spad_ptr, (const uint8_t *)dst_spad_ptr, src1_spad_ptr, nc);       // res = res * g
656        }
657
658        dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr(data_dst + (ir * dst_row_size), dst_spad), dst_row_size,
659                                   dst_row_size_aligned, block_size);
660
661        // prefetch N+2 loop iteration if any
662        const uint32_t pref_block = (ir + BLOCK * 2);
663        if (pref_block < src0_end_row) {
664            const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block);
665            dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src0_spad, data_src0 + (pref_block * src0_row_size)),
666                                       src0_row_size_aligned, src0_row_size, pref_block_size);
667            dma_queue_push_ddr_to_vtcm(dma_queue, dma_make_ptr(src1_spad, data_src1 + (pref_block * src1_row_size)),
668                                       src1_row_size_aligned, src1_row_size, pref_block_size);
669        }
670    }
671
672    dma_queue_flush(dma_queue);
673
674    t2 = HAP_perf_get_qtimer_count();
675
676    FARF(HIGH, "geglu-f32 %d/%d: %ux%ux%ux%u (%u:%u) x %ux%ux%ux%u -> %ux%ux%ux%u usec %u\n", ith, nth,
677         ne00, ne01, ne02, ne03, src0_start_row, src0_end_row, ne10, ne11, ne12, ne13, ne0, ne1, ne2, ne3,
678         (unsigned) HAP_perf_qtimer_count_to_us(t2 - t1));
679}
680
681static void unary_silu_f32(unsigned int n, unsigned int i, void * data) {
682    struct htp_ops_context * octx = (struct htp_ops_context *) data;
683    unary_silu_f32_per_thread(&octx->src0, &octx->dst, octx->op_params, &octx->src0_spad, &octx->dst_spad, n, i,
684                               octx->src0_nrows_per_thread, octx->ctx->dma[i]);
685}
686
687static void glu_swiglu_f32(unsigned int n, unsigned int i, void * data) {
688    struct htp_ops_context * octx = (struct htp_ops_context *) data;
689    glu_swiglu_f32_per_thread(&octx->src0, &octx->src1, &octx->dst, octx->op_params, &octx->src0_spad,
690                               &octx->src1_spad, &octx->dst_spad, n, i, octx->src0_nrows_per_thread, octx->ctx->dma[i]);
691}
692
693static void glu_swiglu_oai_f32(unsigned int n, unsigned int i, void * data) {
694    struct htp_ops_context * octx = (struct htp_ops_context *) data;
695    glu_swiglu_oai_f32_per_thread(&octx->src0, &octx->src1, &octx->dst, octx->op_params, &octx->src0_spad,
696                                   &octx->src1_spad, &octx->dst_spad, n, i, octx->src0_nrows_per_thread, octx->ctx->dma[i]);
697}
698
699static void glu_geglu_f32(unsigned int n, unsigned int i, void * data) {
700    struct htp_ops_context * octx = (struct htp_ops_context *) data;
701    glu_geglu_f32_per_thread(&octx->src0, &octx->src1, &octx->dst, octx->op_params, &octx->src0_spad,
702                               &octx->src1_spad, &octx->dst_spad, n, i, octx->src0_nrows_per_thread, octx->ctx->dma[i]);
703}
704
705static int execute_op_activations_f32(struct htp_ops_context * octx) {
706    int err = HTP_STATUS_OK;
707
708    const struct htp_tensor * src0 = &octx->src0;
709    const struct htp_tensor * src1 = &octx->src1;
710    struct htp_tensor *       dst  = &octx->dst;
711
712    if (((src0->ne[0] * SIZEOF_FP32) != src0->nb[1]) || ((dst->ne[0] * SIZEOF_FP32) != dst->nb[1])) {
713        FARF(ERROR, "Non-contiguous tensors are not supported at this time \n");
714        return HTP_STATUS_NO_SUPPORT;
715    }
716
717    worker_callback_t act_op_func;
718    const char *      op_type = NULL;
719
720    switch (octx->op) {
721        case HTP_OP_UNARY_SILU:
722            act_op_func = unary_silu_f32;
723            op_type     = "silu-f32";
724            break;
725
726        case HTP_OP_GLU_SWIGLU:
727            act_op_func = glu_swiglu_f32;
728            op_type     = "swiglu-f32";
729            break;
730
731        case HTP_OP_GLU_SWIGLU_OAI:
732            act_op_func = glu_swiglu_oai_f32;
733            op_type     = "swiglu-oai-f32";
734            break;
735        case HTP_OP_UNARY_GELU:
736            act_op_func = unary_gelu_f32;
737            op_type     = "gelu-f32";
738            break;
739
740        case HTP_OP_GLU_GEGLU:
741            act_op_func = glu_geglu_f32;
742            op_type     = "geglu-f32";
743            break;
744        default:
745            FARF(ERROR, "Unsupported activations Op %u\n", octx->op);
746            return HTP_STATUS_NO_SUPPORT;
747    }
748
749    const uint32_t n_threads  = octx->n_threads;
750    const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
751
752    size_t src0_row_size = src0->nb[1];
753    size_t src1_row_size = src1->nb[1]; // zero bytes if src1 is not used
754    size_t dst_row_size  = dst->nb[1];
755
756    const bool src1_valid = src1->ne[0];
757    if (!src1_valid) {
758        src1_row_size = src0_row_size;
759    }
760
761    const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);
762    const size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN);
763    const size_t dst_row_size_aligned  = hex_round_up(dst_row_size, VLEN);
764    // VTCM scratchpads for all tensors
765    // N rows per thread, padded to HVX vector size
766
767    size_t spad_size_per_row   = (src0_row_size_aligned + src1_row_size_aligned) + dst_row_size_aligned;
768    size_t vtcm_row_per_thread = (octx->ctx->vtcm_size)/ (n_threads* spad_size_per_row);
769
770    // Make sure the reserved vtcm size is sufficient
771    if(vtcm_row_per_thread ==0){
772        FARF(ERROR, "act-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size,
773             spad_size_per_row * n_threads);
774        return HTP_STATUS_VTCM_TOO_SMALL;
775    }
776
777    octx->src0_spad.size_per_thread = src0_row_size_aligned * vtcm_row_per_thread;
778    octx->src1_spad.size_per_thread = src1_row_size_aligned * vtcm_row_per_thread;
779    octx->dst_spad.size_per_thread  = dst_row_size_aligned * vtcm_row_per_thread;
780
781    octx->dst_spad.size  = n_threads* octx->dst_spad.size_per_thread;
782    octx->src0_spad.size = n_threads* octx->src0_spad.size_per_thread;
783    octx->src1_spad.size = n_threads* octx->src1_spad.size_per_thread;
784
785    octx->src0_spad.data = octx->ctx->vtcm_base;
786    octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size;
787    octx->dst_spad.data  = octx->src1_spad.data + octx->src1_spad.size;
788
789    if (src1->ne[0]) {
790        FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-spad-size %u src1-spad-size %u dst-spad-size %u\n",
791             op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2],
792             src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], octx->src0_spad.size, octx->src1_spad.size,
793             octx->dst_spad.size);
794    } else {
795        FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-spad-size %u src1-spad-size %u dst-spad-size %u\n", op_type,
796             src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
797             octx->src0_spad.size, octx->src1_spad.size, octx->dst_spad.size);
798    }
799
800    if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) {
801        uint32_t n_jobs = MIN(n_threads, src0_nrows);
802        octx->src0_nrows_per_thread = (src0_nrows + n_jobs - 1) / n_jobs;
803        worker_pool_run_func(octx->ctx->worker_pool, act_op_func, octx, n_jobs);
804    }
805
806    return err;
807}
808
809int op_activations(struct htp_ops_context * octx) {
810    int err = HTP_STATUS_OK;
811
812    switch (octx->src0.type) {
813        case HTP_TYPE_F32:
814            err = execute_op_activations_f32(octx);
815            break;
816
817        default:
818            err = HTP_STATUS_NO_SUPPORT;
819            break;
820    }
821
822    return err;
823}