1#include "models.h"
 2
 3ggml_cgraph * clip_graph_internvl::build() {
 4    GGML_ASSERT(model.class_embedding != nullptr);
 5    GGML_ASSERT(model.position_embeddings != nullptr);
 6
 7    const int n_pos = n_patches + 1;
 8    ggml_tensor * inp = build_inp();
 9
10    // add CLS token
11    inp = ggml_concat(ctx0, inp, model.class_embedding, 1);
12
13    // The larger models use a different ViT, which uses RMS norm instead of layer norm
14    // ref: https://github.com/ggml-org/llama.cpp/pull/13443#issuecomment-2869786188
15    norm_type norm_t = (hparams.n_embd == 3200 && hparams.n_layer == 45)
16        ? NORM_TYPE_RMS // 6B ViT (Used by InternVL 2.5/3 - 26B, 38B, 78B)
17        : NORM_TYPE_NORMAL; // 300M ViT (Used by all smaller InternVL models)
18
19    ggml_tensor * cur = build_vit(
20                            inp, n_pos,
21                            norm_t,
22                            hparams.ffn_op,
23                            model.position_embeddings,
24                            nullptr);
25
26    // remove CLS token
27    cur = ggml_view_2d(ctx0, cur,
28        n_embd, n_patches,
29        ggml_row_size(cur->type, n_embd), 0);
30
31    // pixel shuffle
32    {
33        const int scale_factor = model.hparams.n_merge;
34        const int bsz    = 1; // batch size, always 1 for now since we don't support batching
35        const int height = n_patches_y;
36        const int width  = n_patches_x;
37        GGML_ASSERT(scale_factor > 0);
38        cur = ggml_reshape_4d(ctx0, cur, n_embd * scale_factor, height / scale_factor, width, bsz);
39        cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
40        cur = ggml_cont_4d(ctx0, cur,
41            n_embd * scale_factor * scale_factor,
42            height / scale_factor,
43            width / scale_factor,
44            bsz);
45        cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);
46        // flatten to 2D
47        cur = ggml_cont_2d(ctx0, cur,
48            n_embd * scale_factor * scale_factor,
49            cur->ne[1] * cur->ne[2]);
50    }
51
52    // projector (always using GELU activation)
53    {
54        // projector LayerNorm uses pytorch's default eps = 1e-5
55        // ref: https://huggingface.co/OpenGVLab/InternVL3-8B-Instruct/blob/a34d3e4e129a5856abfd6aa6de79776484caa14e/modeling_internvl_chat.py#L79
56        cur = build_norm(cur, model.mm_0_w, model.mm_0_b, NORM_TYPE_NORMAL, 1e-5, -1);
57        cur = build_ffn(cur,
58            model.mm_1_w, model.mm_1_b,
59            nullptr, nullptr,
60            model.mm_3_w, model.mm_3_b,
61            FFN_GELU,
62            -1);
63    }
64
65    // build the graph
66    ggml_build_forward_expand(gf, cur);
67
68    return gf;
69}