1import argparse
 2import os
 3import torch
 4from transformers import AutoModel, AutoTokenizer
 5
 6ap = argparse.ArgumentParser()
 7ap.add_argument("-m", "--model", help="Path to MiniCPM-V model")
 8args = ap.parse_args()
 9
10# find the model part that includes the the multimodal projector weights
11model = AutoModel.from_pretrained(args.model, trust_remote_code=True, local_files_only=True, torch_dtype=torch.bfloat16)
12checkpoint = model.state_dict()
13
14# get a list of mm tensor names
15mm_tensors = [k for k, v in checkpoint.items() if k.startswith("resampler")]
16
17# store these tensors in a new dictionary and torch.save them
18projector = {name: checkpoint[name].float() for name in mm_tensors}
19if 'resampler.proj' in projector.keys() and hasattr(model.llm.config,'scale_emb') is True:
20    projector['resampler.proj'] = projector['resampler.proj'] / model.llm.config.scale_emb
21torch.save(projector, f"{args.model}/minicpmv.projector")
22
23clip_tensors = [k for k, v in checkpoint.items() if k.startswith("vpm")]
24if len(clip_tensors) > 0:
25    clip = {name.replace("vpm.", ""): checkpoint[name].float() for name in clip_tensors}
26    torch.save(clip, f"{args.model}/minicpmv.clip")
27
28    # added tokens should be removed to be able to convert Mistral models
29    if os.path.exists(f"{args.model}/added_tokens.json"):
30        with open(f"{args.model}/added_tokens.json", "w") as f:
31            f.write("{}\n")
32
33config = model.llm.config
34config.auto_map = {
35    "AutoConfig": "configuration_minicpm.MiniCPMConfig",
36    "AutoModel": "modeling_minicpm.MiniCPMModel",
37    "AutoModelForCausalLM": "modeling_minicpm.MiniCPMForCausalLM",
38    "AutoModelForSeq2SeqLM": "modeling_minicpm.MiniCPMForCausalLM",
39    "AutoModelForSequenceClassification": "modeling_minicpm.MiniCPMForSequenceClassification"
40}
41model.llm.save_pretrained(f"{args.model}/model")
42tok = AutoTokenizer.from_pretrained(args.model, trust_remote_code=True)
43tok.save_pretrained(f"{args.model}/model")
44
45print("Done!")
46print(f"Now you can convert {args.model} to a regular LLaMA GGUF file.")
47print(f"Also, use {args.model}/minicpmv.projector to prepare a minicpmv-encoder.gguf file.")