1#pragma once
2
3#include "ggml.h"
4#include "gguf.h"
5#include "clip.h"
6
7#include <climits>
8#include <cstdarg>
9#include <cinttypes>
10#include <string>
11#include <map>
12#include <sstream>
13#include <vector>
14#include <memory>
15
16// Internal header for clip.cpp
17
18#define MTMD_INTERNAL_HEADER
19
20#define KEY_FTYPE "general.file_type"
21#define KEY_NAME "general.name"
22#define KEY_DESCRIPTION "general.description"
23#define KEY_PROJ_TYPE "clip.projector_type"
24#define KEY_HAS_AUDIO_ENC "clip.has_audio_encoder"
25#define KEY_HAS_VISION_ENC "clip.has_vision_encoder"
26#define KEY_USE_GELU "clip.use_gelu"
27#define KEY_USE_SILU "clip.use_silu"
28
29#define KEY_N_EMBD "clip.%s.embedding_length"
30#define KEY_N_FF "clip.%s.feed_forward_length"
31#define KEY_N_BLOCK "clip.%s.block_count"
32#define KEY_PROJ_DIM "clip.%s.projection_dim"
33#define KEY_N_HEAD "clip.%s.attention.head_count"
34#define KEY_LAYER_NORM_EPS "clip.%s.attention.layer_norm_epsilon"
35
36// vision-specific
37#define KEY_VISION_PROJ_TYPE "clip.vision.projector_type" // for models with mixed modalities
38#define KEY_IMAGE_SIZE "clip.vision.image_size"
39#define KEY_IMAGE_MIN_PIXELS "clip.vision.image_min_pixels"
40#define KEY_IMAGE_MAX_PIXELS "clip.vision.image_max_pixels"
41#define KEY_PREPROC_IMAGE_SIZE "clip.vision.preproc_image_size"
42#define KEY_PATCH_SIZE "clip.vision.patch_size"
43#define KEY_IMAGE_MEAN "clip.vision.image_mean"
44#define KEY_IMAGE_STD "clip.vision.image_std"
45#define KEY_FEATURE_LAYER "clip.vision.feature_layer"
46#define KEY_PROJ_SCALE_FACTOR "clip.vision.projector.scale_factor"
47#define KEY_SPATIAL_MERGE_SIZE "clip.vision.spatial_merge_size"
48#define KEY_IS_DEEPSTACK_LAYERS "clip.vision.is_deepstack_layers"
49
50#define KEY_MM_PATCH_MERGE_TYPE "clip.vision.mm_patch_merge_type"
51#define KEY_IMAGE_GRID_PINPOINTS "clip.vision.image_grid_pinpoints"
52#define KEY_IMAGE_CROP_RESOLUTION "clip.vision.image_crop_resolution"
53#define KEY_WIN_ATTN_PATTERN "clip.vision.n_wa_pattern"
54#define KEY_WIN_ATTN_LAYER_INDEXES "clip.vision.wa_layer_indexes"
55#define KEY_ATTN_WINDOW_SIZE "clip.vision.window_size"
56#define KEY_MINICPMV_VERSION "clip.minicpmv_version"
57#define KEY_MINICPMV_QUERY_NUM "clip.minicpmv_query_num"
58
59// audio-specific
60#define KEY_AUDIO_PROJ_TYPE "clip.audio.projector_type" // for models with mixed modalities
61#define KEY_A_NUM_MEL_BINS "clip.audio.num_mel_bins"
62#define KEY_A_PROJ_STACK_FACTOR "clip.audio.projector.stack_factor"
63
64
65//
66// tensor name constants
67//
68
69#define TN_POS_EMBD "%s.position_embd.weight"
70#define TN_CLASS_EMBD "v.class_embd"
71#define TN_PATCH_EMBD "v.patch_embd.weight" // not rename tensor with ".0" postfix for backwrad compat
72#define TN_PATCH_EMBD_1 "v.patch_embd.weight.1"
73#define TN_PATCH_BIAS "v.patch_embd.bias"
74#define TN_NORM_EMBD "v.norm_embd.%s"
75#define TN_ATTN_QKV "%s.blk.%d.attn_qkv.%s"
76#define TN_ATTN_K "%s.blk.%d.attn_k.%s"
77#define TN_ATTN_Q "%s.blk.%d.attn_q.%s"
78#define TN_ATTN_V "%s.blk.%d.attn_v.%s"
79#define TN_ATTN_OUTPUT "%s.blk.%d.attn_out.%s"
80#define TN_ATTN_K_NORM "%s.blk.%d.attn_k_norm.%s"
81#define TN_ATTN_Q_NORM "%s.blk.%d.attn_q_norm.%s"
82#define TN_FFN_DOWN "%s.blk.%d.ffn_down.%s"
83#define TN_FFN_GATE "%s.blk.%d.ffn_gate.%s"
84#define TN_FFN_UP "%s.blk.%d.ffn_up.%s"
85#define TN_FFN_GATE "%s.blk.%d.ffn_gate.%s"
86#define TN_LN_1 "%s.blk.%d.ln1.%s" // layer norm
87#define TN_LN_2 "%s.blk.%d.ln2.%s" // layer norm
88#define TN_LS_1 "%s.blk.%d.ls1.%s" // layer scale
89#define TN_LS_2 "%s.blk.%d.ls2.%s" // layer scale
90#define TN_LN_PRE "%s.pre_ln.%s"
91#define TN_LN_POST "%s.post_ln.%s"
92#define TN_LLAVA_PROJ "mm.%d.%s"
93#define TN_MM_UP "mm.up.%s"
94#define TN_MM_GATE "mm.gate.%s"
95#define TN_MM_DOWN "mm.down.%s"
96#define TN_MM_POST_NORM "mm.post_norm.%s"
97#define TN_MVLM_PROJ_MLP "mm.model.mlp.%d.%s"
98#define TN_MVLM_PROJ_BLOCK "mm.model.mb_block.%d.block.%d.%s"
99#define TN_MVLM_PROJ_PEG "mm.model.peg.%d.%s"
100#define TN_IMAGE_NEWLINE "model.image_newline"
101#define TN_MM_INP_NORM "mm.input_norm.weight"
102#define TN_MM_INP_NORM_B "mm.input_norm.bias"
103#define TN_MM_INP_PROJ "mm.input_projection.weight" // gemma3
104#define TN_MM_SOFT_EMB_N "mm.soft_emb_norm.weight" // gemma3
105#define TN_MM_PROJECTOR "mm.model.fc.weight" // idefics3
106#define TN_MM_PATCH_MERGER "mm.patch_merger.%s" // mistral small 3.1, glm4v
107#define TN_TOK_IMG_BREAK "v.token_embd.img_break" // pixtral
108#define TN_TOK_GLM_BOI "adapter.boi" // glm-edge (these embeddings are not in text model)
109#define TN_TOK_GLM_EOI "adapter.eoi" // glm-edge (these embeddings are not in text model)
110#define TN_DEEPSTACK_NORM "v.deepstack.%d.norm.%s" // qwen3vl deepstack
111#define TN_DEEPSTACK_FC1 "v.deepstack.%d.fc1.%s" // qwen3vl deepstack
112#define TN_DEEPSTACK_FC2 "v.deepstack.%d.fc2.%s" // qwen3vl deepstack
113
114// mimicpmv
115#define TN_MINICPMV_POS_EMBD_K "resampler.pos_embed_k"
116#define TN_MINICPMV_QUERY "resampler.query"
117#define TN_MINICPMV_PROJ "resampler.proj.weight"
118#define TN_MINICPMV_KV_PROJ "resampler.kv.weight"
119#define TN_MINICPMV_ATTN "resampler.attn.%s.%s"
120#define TN_MINICPMV_LN "resampler.ln_%s.%s"
121
122#define TN_GLM_ADAPER_CONV "adapter.conv.%s"
123#define TN_GLM_ADAPTER_LINEAR "adapter.linear.linear.%s"
124#define TN_GLM_ADAPTER_NORM_1 "adapter.linear.norm1.%s"
125#define TN_GLM_ADAPTER_D_H_2_4H "adapter.linear.dense_h_to_4h.%s"
126#define TN_GLM_ADAPTER_GATE "adapter.linear.gate.%s"
127#define TN_GLM_ADAPTER_D_4H_2_H "adapter.linear.dense_4h_to_h.%s"
128
129// ultravox
130#define TN_CONV1D "a.conv1d.%d.%s"
131#define TN_MM_AUDIO_MLP "mm.a.mlp.%d.%s"
132#define TN_MM_AUDIO_FC "mm.a.fc.%s" // fully connected layer
133#define TN_MM_NORM_PRE "mm.a.norm_pre.%s"
134#define TN_MM_NORM_MID "mm.a.norm_mid.%s"
135
136// cogvlm
137#define TN_MM_POST_FC_NORM "mm.post_fc_norm.%s"
138#define TN_MM_H_TO_4H "mm.up.%s"
139#define TN_MM_GATE "mm.gate.%s"
140#define TN_MM_4H_TO_H "mm.down.%s"
141#define TN_TOK_BOI "v.boi"
142#define TN_TOK_EOI "v.eoi"
143
144// (conformer) lfm2
145#define TN_PRE_ENCODE_OUT "a.pre_encode.out.%s"
146#define TN_FFN_NORM "%s.blk.%d.ffn_norm.%s"
147#define TN_FFN_NORM_1 "%s.blk.%d.ffn_norm_1.%s"
148#define TN_FFN_UP_1 "%s.blk.%d.ffn_up_1.%s"
149#define TN_FFN_DOWN_1 "%s.blk.%d.ffn_down_1.%s"
150#define TN_POS_BIAS_U "%s.blk.%d.pos_bias_u"
151#define TN_POS_BIAS_V "%s.blk.%d.pos_bias_v"
152#define TN_NORM_CONV "%s.blk.%d.norm_conv.%s"
153#define TN_LINEAR_POS "%s.blk.%d.linear_pos.%s"
154#define TN_CONV_DW "%s.blk.%d.conv_dw.%s"
155#define TN_CONV_NORM "%s.blk.%d.conv_norm.%s"
156#define TN_CONV_PW1 "%s.blk.%d.conv_pw1.%s"
157#define TN_CONV_PW2 "%s.blk.%d.conv_pw2.%s"
158
159// mobilenetv5 (gemma3n) definitions
160#define TN_MNV5_STEM_CONV "v.conv_stem.conv.weight"
161#define TN_MNV5_STEM_BIAS "v.conv_stem.conv.bias"
162#define TN_MNV5_STEM_BN "v.conv_stem.bn.weight"
163
164// Stage 0 Block (Edge Residual)
165#define TN_MNV5_BLK_S0_EXP_W "v.blk.%d.%d.conv_exp.weight"
166#define TN_MNV5_BLK_S0_BN1_W "v.blk.%d.%d.bn1.weight"
167#define TN_MNV5_BLK_S0_PWL_W "v.blk.%d.%d.conv_pwl.weight"
168#define TN_MNV5_BLK_S0_BN2_W "v.blk.%d.%d.bn2.weight"
169
170// Stage 1+ Block (Universal Inverted Residual)
171#define TN_MNV5_BLK_DW_START_W "v.blk.%d.%d.dw_start.conv.weight"
172#define TN_MNV5_BLK_DW_START_BN "v.blk.%d.%d.dw_start.bn.weight"
173#define TN_MNV5_BLK_DW_MID_W "v.blk.%d.%d.dw_mid.conv.weight"
174#define TN_MNV5_BLK_DW_MID_BN "v.blk.%d.%d.dw_mid.bn.weight"
175#define TN_MNV5_BLK_PW_EXP_W "v.blk.%d.%d.pw_exp.conv.weight"
176#define TN_MNV5_BLK_PW_EXP_BN "v.blk.%d.%d.pw_exp.bn.weight"
177#define TN_MNV5_BLK_PW_PROJ_W "v.blk.%d.%d.pw_proj.conv.weight"
178#define TN_MNV5_BLK_PW_PROJ_BN "v.blk.%d.%d.pw_proj.bn.weight"
179#define TN_MNV5_BLK_LAYER_SCALE "v.blk.%d.%d.layer_scale.gamma"
180
181// Attention Components
182#define TN_MNV5_ATTN_Q_W "v.blk.%d.%d.attn.query.proj.weight"
183#define TN_MNV5_ATTN_K_W "v.blk.%d.%d.attn.key.proj.weight"
184#define TN_MNV5_ATTN_V_W "v.blk.%d.%d.attn.value.proj.weight"
185#define TN_MNV5_ATTN_O_W "v.blk.%d.%d.attn.output.proj.weight"
186#define TN_MNV5_ATTN_K_DW "v.blk.%d.%d.attn.key.down_conv.weight"
187#define TN_MNV5_ATTN_K_NORM "v.blk.%d.%d.attn.key.norm.weight"
188#define TN_MNV5_ATTN_V_DW "v.blk.%d.%d.attn.value.down_conv.weight"
189#define TN_MNV5_ATTN_V_NORM "v.blk.%d.%d.attn.value.norm.weight"
190#define TN_MNV5_ATTN_NORM "v.blk.%d.%d.norm.weight" // Block norm used in attn blocks
191
192// MSFA
193#define TN_MNV5_MSFA_FFN_EXP_W "v.msfa.ffn.pw_exp.conv.weight"
194#define TN_MNV5_MSFA_FFN_EXP_BN "v.msfa.ffn.pw_exp.bn.weight"
195#define TN_MNV5_MSFA_FFN_PROJ_W "v.msfa.ffn.pw_proj.conv.weight"
196#define TN_MNV5_MSFA_FFN_PROJ_BN "v.msfa.ffn.pw_proj.bn.weight"
197#define TN_MNV5_MSFA_NORM "v.msfa.norm.weight"
198
199
200// align x to upper multiple of n
201#define CLIP_ALIGN(x, n) ((((x) + (n) - 1) / (n)) * (n))
202
203// forward declaration
204// TODO: improve this later
205struct clip_ctx;
206
207enum projector_type {
208 PROJECTOR_TYPE_MLP,
209 PROJECTOR_TYPE_MLP_NORM,
210 PROJECTOR_TYPE_LDP,
211 PROJECTOR_TYPE_LDPV2,
212 PROJECTOR_TYPE_MINICPMV,
213 PROJECTOR_TYPE_GLM_EDGE,
214 PROJECTOR_TYPE_QWEN2VL,
215 PROJECTOR_TYPE_QWEN3VL,
216 PROJECTOR_TYPE_GEMMA3,
217 PROJECTOR_TYPE_GEMMA3NV,
218 PROJECTOR_TYPE_GEMMA3NA,
219 PROJECTOR_TYPE_IDEFICS3,
220 PROJECTOR_TYPE_PIXTRAL,
221 PROJECTOR_TYPE_QWEN25VL,
222 PROJECTOR_TYPE_ULTRAVOX,
223 PROJECTOR_TYPE_INTERNVL,
224 PROJECTOR_TYPE_LLAMA4,
225 PROJECTOR_TYPE_QWEN2A,
226 PROJECTOR_TYPE_GLMA,
227 PROJECTOR_TYPE_QWEN25O, // will be replaced by QWEN2A or QWEN25VL depending on clip_ctx
228 PROJECTOR_TYPE_VOXTRAL,
229 PROJECTOR_TYPE_MUSIC_FLAMINGO,
230 PROJECTOR_TYPE_LFM2,
231 PROJECTOR_TYPE_KIMIVL,
232 PROJECTOR_TYPE_LIGHTONOCR,
233 PROJECTOR_TYPE_COGVLM,
234 PROJECTOR_TYPE_JANUS_PRO,
235 PROJECTOR_TYPE_LFM2A,
236 PROJECTOR_TYPE_GLM4V,
237 PROJECTOR_TYPE_YOUTUVL,
238 PROJECTOR_TYPE_KIMIK25,
239 PROJECTOR_TYPE_UNKNOWN,
240};
241
242static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
243 { PROJECTOR_TYPE_MLP, "mlp" },
244 { PROJECTOR_TYPE_LDP, "ldp" },
245 { PROJECTOR_TYPE_LDPV2, "ldpv2"},
246 { PROJECTOR_TYPE_MINICPMV, "resampler"},
247 { PROJECTOR_TYPE_GLM_EDGE, "adapter"},
248 { PROJECTOR_TYPE_QWEN2VL, "qwen2vl_merger"},
249 { PROJECTOR_TYPE_QWEN25VL, "qwen2.5vl_merger"},
250 { PROJECTOR_TYPE_QWEN3VL, "qwen3vl_merger"},
251 { PROJECTOR_TYPE_GEMMA3, "gemma3"},
252 { PROJECTOR_TYPE_GEMMA3NV, "gemma3nv"},
253 { PROJECTOR_TYPE_GEMMA3NA, "gemma3na"},
254 { PROJECTOR_TYPE_IDEFICS3, "idefics3"},
255 { PROJECTOR_TYPE_PIXTRAL, "pixtral"},
256 { PROJECTOR_TYPE_ULTRAVOX, "ultravox"},
257 { PROJECTOR_TYPE_INTERNVL, "internvl"},
258 { PROJECTOR_TYPE_LLAMA4, "llama4"},
259 { PROJECTOR_TYPE_QWEN2A, "qwen2a"},
260 { PROJECTOR_TYPE_GLMA, "glma"},
261 { PROJECTOR_TYPE_QWEN25O, "qwen2.5o"},
262 { PROJECTOR_TYPE_VOXTRAL, "voxtral"},
263 { PROJECTOR_TYPE_MUSIC_FLAMINGO, "musicflamingo"},
264 { PROJECTOR_TYPE_LFM2, "lfm2"},
265 { PROJECTOR_TYPE_KIMIVL, "kimivl"},
266 { PROJECTOR_TYPE_LIGHTONOCR,"lightonocr"},
267 { PROJECTOR_TYPE_COGVLM, "cogvlm"},
268 { PROJECTOR_TYPE_JANUS_PRO, "janus_pro"},
269 { PROJECTOR_TYPE_LFM2A, "lfm2a"},
270 { PROJECTOR_TYPE_GLM4V, "glm4v"},
271 { PROJECTOR_TYPE_YOUTUVL, "youtuvl"},
272 { PROJECTOR_TYPE_KIMIK25, "kimik25"},
273};
274
275static projector_type clip_projector_type_from_string(const std::string & str) {
276 for (const auto & pair : PROJECTOR_TYPE_NAMES) {
277 if (pair.second == str) {
278 return pair.first;
279 }
280 }
281 return PROJECTOR_TYPE_UNKNOWN;
282}
283
284// RGB uint8 image
285struct clip_image_u8 {
286 int nx;
287 int ny;
288
289 std::vector<uint8_t> buf;
290};
291
292// For images, buf.size() == nx*ny*3
293// Memory layout: RGBRGBRGB...
294// For audio, only one channel is used, buf.size() == nx*ny
295// nx will be n_frames and ny will be n_mel
296struct clip_image_f32 {
297 int nx;
298 int ny;
299
300 std::vector<float> buf;
301};
302
303//
304// logging
305//
306
307static void clip_log_callback_default(enum ggml_log_level level, const char * text, void * user_data) {
308 (void) level;
309 (void) user_data;
310 fputs(text, stderr);
311 fflush(stderr);
312}
313
314struct clip_logger_state {
315 ggml_log_callback log_callback;
316 void * log_callback_user_data;
317};
318
319extern struct clip_logger_state g_logger_state;
320
321static void clip_log_internal_v(enum ggml_log_level level, const char * format, va_list args) {
322 if (format == NULL) {
323 return;
324 }
325 va_list args_copy;
326 va_copy(args_copy, args);
327 char buffer[128];
328 int len = vsnprintf(buffer, 128, format, args);
329 if (len < 128) {
330 g_logger_state.log_callback(level, buffer, g_logger_state.log_callback_user_data);
331 } else {
332 char * buffer2 = (char *) calloc(len + 1, sizeof(char));
333 vsnprintf(buffer2, len + 1, format, args_copy);
334 buffer2[len] = 0;
335 g_logger_state.log_callback(level, buffer2, g_logger_state.log_callback_user_data);
336 free(buffer2);
337 }
338 va_end(args_copy);
339}
340
341static void clip_log_internal(enum ggml_log_level level, const char * format, ...) {
342 va_list args;
343 va_start(args, format);
344 clip_log_internal_v(level, format, args);
345 va_end(args);
346}
347
348#define LOG_INF(...) clip_log_internal(GGML_LOG_LEVEL_INFO, __VA_ARGS__)
349#define LOG_WRN(...) clip_log_internal(GGML_LOG_LEVEL_WARN, __VA_ARGS__)
350#define LOG_ERR(...) clip_log_internal(GGML_LOG_LEVEL_ERROR, __VA_ARGS__)
351#define LOG_DBG(...) clip_log_internal(GGML_LOG_LEVEL_DEBUG, __VA_ARGS__)
352#define LOG_CNT(...) clip_log_internal(GGML_LOG_LEVEL_CONT, __VA_ARGS__)
353
354//
355// cpp wrappers
356//
357
358// wrapper for clip_image_size
359struct clip_image_size_deleter {
360 void operator()(clip_image_size * val) { clip_image_size_free(val); }
361};
362typedef std::unique_ptr<clip_image_size, clip_image_size_deleter> clip_image_size_ptr;
363
364// wrapper for clip_image_u8
365struct clip_image_u8_deleter {
366 void operator()(clip_image_u8 * val) { clip_image_u8_free(val); }
367};
368typedef std::unique_ptr<clip_image_u8, clip_image_u8_deleter> clip_image_u8_ptr;
369
370// wrapper for clip_image_f32
371struct clip_image_f32_deleter {
372 void operator()(clip_image_f32 * val) { clip_image_f32_free(val); }
373};
374typedef std::unique_ptr<clip_image_f32, clip_image_f32_deleter> clip_image_f32_ptr;
375
376struct clip_image_u8_batch {
377 std::vector<clip_image_u8_ptr> entries;
378};
379
380struct clip_image_f32_batch {
381 std::vector<clip_image_f32_ptr> entries;
382 bool is_audio = false;
383
384 // for llava-uhd style models, we need to know the grid size
385 // note: entries.size() == grid_x * grid_y + 1 (one overview image)
386 int grid_x = 0;
387 int grid_y = 0;
388
389 clip_image_f32_batch clone() const {
390 clip_image_f32_batch new_batch{
391 /* entries */ {},
392 /* is_audio */ is_audio,
393 /* grid_x */ grid_x,
394 /* grid_y */ grid_y,
395 };
396 new_batch.entries.reserve(entries.size());
397 for (const auto & entry : entries) {
398 new_batch.entries.emplace_back(new clip_image_f32(*entry));
399 }
400 return new_batch;
401 }
402};
403
404//
405// common utils
406//
407
408static std::string string_format(const char * fmt, ...) {
409 va_list ap;
410 va_list ap2;
411 va_start(ap, fmt);
412 va_copy(ap2, ap);
413 int size = vsnprintf(NULL, 0, fmt, ap);
414 GGML_ASSERT(size >= 0 && size < INT_MAX); // NOLINT
415 std::vector<char> buf(size + 1);
416 int size2 = vsnprintf(buf.data(), size + 1, fmt, ap2);
417 GGML_ASSERT(size2 == size);
418 va_end(ap2);
419 va_end(ap);
420 return std::string(buf.data(), buf.size());
421}
422
423static void string_replace_all(std::string & s, const std::string & search, const std::string & replace) {
424 if (search.empty()) {
425 return;
426 }
427 std::string builder;
428 builder.reserve(s.length());
429 size_t pos = 0;
430 size_t last_pos = 0;
431 while ((pos = s.find(search, last_pos)) != std::string::npos) {
432 builder.append(s, last_pos, pos - last_pos);
433 builder.append(replace);
434 last_pos = pos + search.length();
435 }
436 builder.append(s, last_pos, std::string::npos);
437 s = std::move(builder);
438}
439
440// split string by a `std::string delim` instead of `char delim`
441static std::vector<std::string> string_split_str(std::string s, const std::string & delimiter) {
442 std::vector<std::string> tokens;
443 size_t pos = 0;
444 std::string token;
445 while ((pos = s.find(delimiter)) != std::string::npos) {
446 token = s.substr(0, pos);
447 tokens.push_back(token);
448 s.erase(0, pos + delimiter.length());
449 }
450 tokens.push_back(s);
451 return tokens;
452}
453
454//
455// gguf utils
456//
457
458static std::string gguf_data_to_str(enum gguf_type type, const void * data, int i) {
459 switch (type) {
460 case GGUF_TYPE_UINT8: return std::to_string(((const uint8_t *)data)[i]);
461 case GGUF_TYPE_INT8: return std::to_string(((const int8_t *)data)[i]);
462 case GGUF_TYPE_UINT16: return std::to_string(((const uint16_t *)data)[i]);
463 case GGUF_TYPE_INT16: return std::to_string(((const int16_t *)data)[i]);
464 case GGUF_TYPE_UINT32: return std::to_string(((const uint32_t *)data)[i]);
465 case GGUF_TYPE_INT32: return std::to_string(((const int32_t *)data)[i]);
466 case GGUF_TYPE_UINT64: return std::to_string(((const uint64_t *)data)[i]);
467 case GGUF_TYPE_INT64: return std::to_string(((const int64_t *)data)[i]);
468 case GGUF_TYPE_FLOAT32: return std::to_string(((const float *)data)[i]);
469 case GGUF_TYPE_FLOAT64: return std::to_string(((const double *)data)[i]);
470 case GGUF_TYPE_BOOL: return ((const bool *)data)[i] ? "true" : "false";
471 default: return string_format("unknown type %d", type);
472 }
473}
474
475static std::string gguf_kv_to_str(const struct gguf_context * ctx_gguf, int i) {
476 const enum gguf_type type = gguf_get_kv_type(ctx_gguf, i);
477
478 switch (type) {
479 case GGUF_TYPE_STRING:
480 return gguf_get_val_str(ctx_gguf, i);
481 case GGUF_TYPE_ARRAY:
482 {
483 const enum gguf_type arr_type = gguf_get_arr_type(ctx_gguf, i);
484 int arr_n = gguf_get_arr_n(ctx_gguf, i);
485 const void * data = arr_type == GGUF_TYPE_STRING ? nullptr : gguf_get_arr_data(ctx_gguf, i);
486 std::stringstream ss;
487 ss << "[";
488 for (int j = 0; j < arr_n; j++) {
489 if (arr_type == GGUF_TYPE_STRING) {
490 std::string val = gguf_get_arr_str(ctx_gguf, i, j);
491 // escape quotes
492 string_replace_all(val, "\\", "\\\\");
493 string_replace_all(val, "\"", "\\\"");
494 ss << '"' << val << '"';
495 } else if (arr_type == GGUF_TYPE_ARRAY) {
496 ss << "???";
497 } else {
498 ss << gguf_data_to_str(arr_type, data, j);
499 }
500 if (j < arr_n - 1) {
501 ss << ", ";
502 }
503 }
504 ss << "]";
505 return ss.str();
506 }
507 default:
508 return gguf_data_to_str(type, gguf_get_val_data(ctx_gguf, i), 0);
509 }
510}
511
512//
513// debugging
514//
515
516static void print_tensor_shape(ggml_tensor * t) {
517 printf("%s.shape = [", t->name);
518 for (int i = 0; i < ggml_n_dims(t); ++i) {
519 printf("%" PRId64, t->ne[i]);
520 if (i < ggml_n_dims(t) - 1) {
521 printf(", ");
522 }
523 }
524 printf("]\n");
525}
526
527static void print_tensor_data(ggml_tensor * t, uint8_t * data, int64_t n) {
528 ggml_type type = t->type;
529 int64_t * ne = t->ne;
530 size_t * nb = t->nb;
531 for (int64_t i3 = 0; i3 < ne[3]; i3++) {
532 printf("%s.data: [\n", t->name);
533 for (int64_t i2 = 0; i2 < ne[2]; i2++) {
534 if (i2 == n && ne[2] > 2*n) {
535 printf(" ..., \n");
536 i2 = ne[2] - n;
537 }
538 printf(" [\n");
539 for (int64_t i1 = 0; i1 < ne[1]; i1++) {
540 if (i1 == n && ne[1] > 2*n) {
541 printf(" ..., \n");
542 i1 = ne[1] - n;
543 }
544 printf(" [");
545 for (int64_t i0 = 0; i0 < ne[0]; i0++) {
546 if (i0 == n && ne[0] > 2*n) {
547 printf("..., ");
548 i0 = ne[0] - n;
549 }
550 size_t i = i3 * nb[3] + i2 * nb[2] + i1 * nb[1] + i0 * nb[0];
551 float v;
552 if (type == GGML_TYPE_F16) {
553 v = ggml_fp16_to_fp32(*(ggml_fp16_t *) &data[i]);
554 } else if (type == GGML_TYPE_F32) {
555 v = *(float *) &data[i];
556 } else if (type == GGML_TYPE_I32) {
557 v = (float) *(int32_t *) &data[i];
558 } else if (type == GGML_TYPE_I16) {
559 v = (float) *(int16_t *) &data[i];
560 } else if (type == GGML_TYPE_I8) {
561 v = (float) *(int8_t *) &data[i];
562 } else {
563 GGML_ABORT("fatal error");
564 }
565 printf("%8.4f", v);
566 if (i0 < ne[0] - 1) printf(", ");
567 }
568 printf("],\n");
569 }
570 printf(" ],\n");
571 }
572 printf(" ]\n");
573 }
574}
575
576void clip_debug_encode(clip_ctx * ctx, int h, int w, float fill_value);
577
578//
579// API used internally with mtmd
580//
581
582projector_type clip_get_projector_type(const struct clip_ctx * ctx);