1#pragma once
  2
  3#include "ggml.h"
  4#include "mtmd.h"
  5
  6#include <stddef.h>
  7#include <stdint.h>
  8
  9// !!! Internal header, to be used by mtmd only !!!
 10
 11#define MTMD_INTERNAL_HEADER
 12
 13struct clip_ctx;
 14
 15struct clip_image_size {
 16    int width;
 17    int height;
 18};
 19
 20struct clip_image_f32;
 21struct clip_image_u8_batch;
 22struct clip_image_f32_batch;
 23
 24enum clip_modality {
 25    CLIP_MODALITY_VISION,
 26    CLIP_MODALITY_AUDIO,
 27};
 28
 29enum clip_flash_attn_type {
 30    CLIP_FLASH_ATTN_TYPE_AUTO     = -1,
 31    CLIP_FLASH_ATTN_TYPE_DISABLED = 0,
 32    CLIP_FLASH_ATTN_TYPE_ENABLED  = 1,
 33};
 34
 35struct clip_context_params {
 36    bool use_gpu;
 37    enum clip_flash_attn_type flash_attn_type;
 38    int image_min_tokens;
 39    int image_max_tokens;
 40    bool warmup;
 41    ggml_backend_sched_eval_callback cb_eval;
 42    void * cb_eval_user_data;
 43};
 44
 45struct clip_init_result {
 46    struct clip_ctx * ctx_v; // vision context
 47    struct clip_ctx * ctx_a; // audio context
 48};
 49
 50struct clip_init_result clip_init(const char * fname, struct clip_context_params ctx_params);
 51
 52void clip_free(struct clip_ctx * ctx);
 53
 54size_t clip_embd_nbytes(const struct clip_ctx * ctx);
 55size_t clip_embd_nbytes_by_img(const struct clip_ctx * ctx, int img_w, int img_h);
 56
 57int32_t clip_get_image_size (const struct clip_ctx * ctx);
 58int32_t clip_get_patch_size (const struct clip_ctx * ctx);
 59int32_t clip_get_hidden_size(const struct clip_ctx * ctx);
 60
 61// TODO: should be enum, not string
 62const char * clip_patch_merge_type(const struct clip_ctx * ctx);
 63
 64int clip_n_output_tokens(const struct clip_ctx * ctx, struct clip_image_f32 * img);
 65
 66// for M-RoPE, this will be the number of token positions in X and Y directions
 67// for other models, X will be the total number of tokens and Y will be 1
 68int clip_n_output_tokens_x(const struct clip_ctx * ctx, struct clip_image_f32 * img);
 69int clip_n_output_tokens_y(const struct clip_ctx * ctx, struct clip_image_f32 * img);
 70
 71// this should be equal to the embedding dimension of the text model
 72int clip_n_mmproj_embd(const struct clip_ctx * ctx);
 73
 74struct clip_image_size      * clip_image_size_init(void);
 75struct clip_image_u8        * clip_image_u8_init (void);
 76struct clip_image_f32       * clip_image_f32_init(void);
 77struct clip_image_f32_batch * clip_image_f32_batch_init(void); // only used by libllava
 78
 79// nx, ny are the output image dimensions
 80unsigned char * clip_image_u8_get_data(struct clip_image_u8 * img, uint32_t * nx, uint32_t * ny);
 81
 82void clip_image_size_free (struct clip_image_size * img_size);
 83void clip_image_u8_free (struct clip_image_u8  * img);
 84void clip_image_f32_free(struct clip_image_f32 * img);
 85void clip_image_u8_batch_free (struct clip_image_u8_batch  * batch);
 86void clip_image_f32_batch_free(struct clip_image_f32_batch * batch);
 87
 88// use for accessing underlay data of clip_image_f32_batch
 89size_t clip_image_f32_batch_n_images(const struct clip_image_f32_batch * batch); // equivalent to batch->size()
 90size_t clip_image_f32_batch_nx(const struct clip_image_f32_batch * batch, int idx); // equivalent to batch[idx]->nx
 91size_t clip_image_f32_batch_ny(const struct clip_image_f32_batch * batch, int idx); // equivalent to batch[idx]->ny
 92struct clip_image_f32 * clip_image_f32_get_img(const struct clip_image_f32_batch * batch, int idx); // equivalent to batch[idx]->data
 93
 94/**
 95 * Build image from pixels decoded by other libraries instead of stb_image.h for better performance.
 96 * The memory layout is RGBRGBRGB..., input buffer length must be 3*nx*ny bytes
 97 */
 98void clip_build_img_from_pixels(const unsigned char * rgb_pixels, int nx, int ny, struct clip_image_u8 * img);
 99
100/** preprocess img and store the result in res_imgs, pad_to_square may be overridden to false depending on model configuration */
101bool clip_image_preprocess(struct clip_ctx * ctx, const struct clip_image_u8 * img, struct clip_image_f32_batch * res_imgs );
102
103struct ggml_tensor * clip_get_newline_tensor(const struct clip_ctx * ctx);
104
105bool clip_image_encode      (struct clip_ctx * ctx, int n_threads, struct clip_image_f32 * img, float * vec);
106bool clip_image_batch_encode(struct clip_ctx * ctx, int n_threads, const struct clip_image_f32_batch * imgs, float * vec);
107
108int clip_is_minicpmv(const struct clip_ctx * ctx);
109bool clip_is_glm(const struct clip_ctx * ctx);
110bool clip_is_llava(const struct clip_ctx * ctx);
111// note for contributor: this clip_is_(model) pattern is deprecated
112//                       do NOT add new functions like this
113
114bool clip_encode_float_image (struct clip_ctx * ctx, int n_threads, float * img, int h, int w, float * vec);
115
116// use by audio input
117void clip_image_f32_batch_add_mel(struct clip_image_f32_batch * batch, int n_mel, int n_frames, float * mel);
118
119bool clip_has_vision_encoder(const struct clip_ctx * ctx);
120bool clip_has_audio_encoder(const struct clip_ctx * ctx);
121bool clip_has_whisper_encoder(const struct clip_ctx * ctx);