1#include "common.h"
2#include "llama.h"
3#include "gguf.h"
4
5#include <cstdio>
6#include <cstring>
7#include <vector>
8#include <string>
9#include <unordered_map>
10#include <map>
11#include <fstream>
12#include <cmath>
13#include <cctype>
14#include <algorithm>
15#include <filesystem>
16
17struct quant_option {
18 std::string name;
19 llama_ftype ftype;
20 std::string desc;
21};
22
23static const std::vector<quant_option> QUANT_OPTIONS = {
24 { "Q4_0", LLAMA_FTYPE_MOSTLY_Q4_0, " 4.34G, +0.4685 ppl @ Llama-3-8B", },
25 { "Q4_1", LLAMA_FTYPE_MOSTLY_Q4_1, " 4.78G, +0.4511 ppl @ Llama-3-8B", },
26 { "MXFP4_MOE",LLAMA_FTYPE_MOSTLY_MXFP4_MOE," MXFP4 MoE", },
27 { "Q5_0", LLAMA_FTYPE_MOSTLY_Q5_0, " 5.21G, +0.1316 ppl @ Llama-3-8B", },
28 { "Q5_1", LLAMA_FTYPE_MOSTLY_Q5_1, " 5.65G, +0.1062 ppl @ Llama-3-8B", },
29 { "IQ2_XXS", LLAMA_FTYPE_MOSTLY_IQ2_XXS, " 2.06 bpw quantization", },
30 { "IQ2_XS", LLAMA_FTYPE_MOSTLY_IQ2_XS, " 2.31 bpw quantization", },
31 { "IQ2_S", LLAMA_FTYPE_MOSTLY_IQ2_S, " 2.5 bpw quantization", },
32 { "IQ2_M", LLAMA_FTYPE_MOSTLY_IQ2_M, " 2.7 bpw quantization", },
33 { "IQ1_S", LLAMA_FTYPE_MOSTLY_IQ1_S, " 1.56 bpw quantization", },
34 { "IQ1_M", LLAMA_FTYPE_MOSTLY_IQ1_M, " 1.75 bpw quantization", },
35 { "TQ1_0", LLAMA_FTYPE_MOSTLY_TQ1_0, " 1.69 bpw ternarization", },
36 { "TQ2_0", LLAMA_FTYPE_MOSTLY_TQ2_0, " 2.06 bpw ternarization", },
37 { "Q2_K", LLAMA_FTYPE_MOSTLY_Q2_K, " 2.96G, +3.5199 ppl @ Llama-3-8B", },
38 { "Q2_K_S", LLAMA_FTYPE_MOSTLY_Q2_K_S, " 2.96G, +3.1836 ppl @ Llama-3-8B", },
39 { "IQ3_XXS", LLAMA_FTYPE_MOSTLY_IQ3_XXS, " 3.06 bpw quantization", },
40 { "IQ3_S", LLAMA_FTYPE_MOSTLY_IQ3_S, " 3.44 bpw quantization", },
41 { "IQ3_M", LLAMA_FTYPE_MOSTLY_IQ3_M, " 3.66 bpw quantization mix", },
42 { "Q3_K", LLAMA_FTYPE_MOSTLY_Q3_K_M, "alias for Q3_K_M" },
43 { "IQ3_XS", LLAMA_FTYPE_MOSTLY_IQ3_XS, " 3.3 bpw quantization", },
44 { "Q3_K_S", LLAMA_FTYPE_MOSTLY_Q3_K_S, " 3.41G, +1.6321 ppl @ Llama-3-8B", },
45 { "Q3_K_M", LLAMA_FTYPE_MOSTLY_Q3_K_M, " 3.74G, +0.6569 ppl @ Llama-3-8B", },
46 { "Q3_K_L", LLAMA_FTYPE_MOSTLY_Q3_K_L, " 4.03G, +0.5562 ppl @ Llama-3-8B", },
47 { "IQ4_NL", LLAMA_FTYPE_MOSTLY_IQ4_NL, " 4.50 bpw non-linear quantization", },
48 { "IQ4_XS", LLAMA_FTYPE_MOSTLY_IQ4_XS, " 4.25 bpw non-linear quantization", },
49 { "Q4_K", LLAMA_FTYPE_MOSTLY_Q4_K_M, "alias for Q4_K_M", },
50 { "Q4_K_S", LLAMA_FTYPE_MOSTLY_Q4_K_S, " 4.37G, +0.2689 ppl @ Llama-3-8B", },
51 { "Q4_K_M", LLAMA_FTYPE_MOSTLY_Q4_K_M, " 4.58G, +0.1754 ppl @ Llama-3-8B", },
52 { "Q5_K", LLAMA_FTYPE_MOSTLY_Q5_K_M, "alias for Q5_K_M", },
53 { "Q5_K_S", LLAMA_FTYPE_MOSTLY_Q5_K_S, " 5.21G, +0.1049 ppl @ Llama-3-8B", },
54 { "Q5_K_M", LLAMA_FTYPE_MOSTLY_Q5_K_M, " 5.33G, +0.0569 ppl @ Llama-3-8B", },
55 { "Q6_K", LLAMA_FTYPE_MOSTLY_Q6_K, " 6.14G, +0.0217 ppl @ Llama-3-8B", },
56 { "Q8_0", LLAMA_FTYPE_MOSTLY_Q8_0, " 7.96G, +0.0026 ppl @ Llama-3-8B", },
57 { "F16", LLAMA_FTYPE_MOSTLY_F16, "14.00G, +0.0020 ppl @ Mistral-7B", },
58 { "BF16", LLAMA_FTYPE_MOSTLY_BF16, "14.00G, -0.0050 ppl @ Mistral-7B", },
59 { "F32", LLAMA_FTYPE_ALL_F32, "26.00G @ 7B", },
60 // Note: Ensure COPY comes after F32 to avoid ftype 0 from matching.
61 { "COPY", LLAMA_FTYPE_ALL_F32, "only copy tensors, no quantizing", },
62};
63
64// Quantization types. Changes to this struct must be replicated in llama-quantize.cpp
65struct tensor_quantization {
66 std::string name;
67 ggml_type quant = GGML_TYPE_COUNT;
68};
69
70static const char * const LLM_KV_QUANTIZE_IMATRIX_FILE = "quantize.imatrix.file";
71static const char * const LLM_KV_QUANTIZE_IMATRIX_DATASET = "quantize.imatrix.dataset";
72static const char * const LLM_KV_QUANTIZE_IMATRIX_N_ENTRIES = "quantize.imatrix.entries_count";
73static const char * const LLM_KV_QUANTIZE_IMATRIX_N_CHUNKS = "quantize.imatrix.chunks_count";
74
75// TODO: share with imatrix.cpp
76static const char * const LLM_KV_IMATRIX_DATASETS = "imatrix.datasets";
77static const char * const LLM_KV_IMATRIX_CHUNK_COUNT = "imatrix.chunk_count";
78static const char * const LLM_KV_IMATRIX_CHUNK_SIZE = "imatrix.chunk_size";
79
80static bool striequals(const char * a, const char * b) {
81 while (*a && *b) {
82 if (std::tolower(*a) != std::tolower(*b)) {
83 return false;
84 }
85 a++; b++;
86 }
87 return *a == *b;
88}
89
90static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftype, std::string & ftype_str_out) {
91 std::string ftype_str;
92
93 for (auto ch : ftype_str_in) {
94 ftype_str.push_back(std::toupper(ch));
95 }
96 for (const auto & it : QUANT_OPTIONS) {
97 if (striequals(it.name.c_str(), ftype_str.c_str())) {
98 ftype = it.ftype;
99 ftype_str_out = it.name;
100 return true;
101 }
102 }
103 try {
104 int ftype_int = std::stoi(ftype_str);
105 for (const auto & it : QUANT_OPTIONS) {
106 if (it.ftype == ftype_int) {
107 ftype = it.ftype;
108 ftype_str_out = it.name;
109 return true;
110 }
111 }
112 }
113 catch (...) {
114 // stoi failed
115 }
116 return false;
117}
118
119[[noreturn]]
120static void usage(const char * executable) {
121 printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--include-weights]\n", executable);
122 printf(" [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--tensor-type] [--tensor-type-file]\n");
123 printf(" [--prune-layers] [--keep-split] [--override-kv]\n");
124 printf(" model-f32.gguf [model-quant.gguf] type [nthreads]\n\n");
125 printf(" --allow-requantize\n");
126 printf(" allow requantizing tensors that have already been quantized\n");
127 printf(" WARNING: this can severely reduce quality compared to quantizing\n");
128 printf(" from 16bit or 32bit!\n");
129 printf(" --leave-output-tensor\n");
130 printf(" leave output.weight un(re)quantized\n");
131 printf(" increases model size but may also increase quality, especially when requantizing\n");
132 printf(" --pure\n");
133 printf(" disable k-quant mixtures and quantize all tensors to the same type\n");
134 printf(" --imatrix file_name\n");
135 printf(" use data in file_name as importance matrix for quant optimizations\n");
136 printf(" --include-weights tensor_name\n");
137 printf(" use importance matrix for this/these tensor(s)\n");
138 printf(" --exclude-weights tensor_name\n");
139 printf(" do not use importance matrix for this/these tensor(s)\n");
140 printf(" --output-tensor-type ggml_type\n");
141 printf(" use this ggml_type for the output.weight tensor\n");
142 printf(" --token-embedding-type ggml_type\n");
143 printf(" use this ggml_type for the token embeddings tensor\n");
144 printf(" --tensor-type tensor_name=ggml_type\n");
145 printf(" quantize this tensor to this ggml_type\n");
146 printf(" this is an advanced option to selectively quantize tensors. may be specified multiple times.\n");
147 printf(" example: --tensor-type attn_q=q8_0\n");
148 printf(" --tensor-type-file tensor_types.txt\n");
149 printf(" list of tensors to quantize to a specific ggml_type\n");
150 printf(" this is an advanced option to selectively quantize a long list of tensors.\n");
151 printf(" the file should use the same format as above, separated by spaces or newlines.\n");
152 printf(" --prune-layers L0,L1,L2...\n");
153 printf(" comma-separated list of layer numbers to prune from the model\n");
154 printf(" WARNING: this is an advanced option, use with care.\n");
155 printf(" --keep-split\n");
156 printf(" generate quantized model in the same shards as input\n");
157 printf(" --override-kv KEY=TYPE:VALUE\n");
158 printf(" override model metadata by key in the quantized model. may be specified multiple times.\n");
159 printf(" WARNING: this is an advanced option, use with care.\n\n");
160 printf("note: --include-weights and --exclude-weights cannot be used together\n\n");
161 printf("-----------------------------------------------------------------------------\n");
162 printf(" allowed quantization types\n");
163 printf("-----------------------------------------------------------------------------\n\n");
164 for (const auto & it : QUANT_OPTIONS) {
165 if (it.name != "COPY") {
166 printf(" %2d or ", it.ftype);
167 } else {
168 printf(" ");
169 }
170 printf("%-7s : %s\n", it.name.c_str(), it.desc.c_str());
171 }
172 exit(1);
173}
174
175static int load_legacy_imatrix(const std::string & imatrix_file, std::vector<std::string> & imatrix_datasets, std::unordered_map<std::string, std::vector<float>> & imatrix_data) {
176 std::ifstream in(imatrix_file.c_str(), std::ios::binary);
177 if (!in) {
178 printf("%s: failed to open %s\n",__func__, imatrix_file.c_str());
179 exit(1);
180 }
181 int n_entries;
182 in.read((char *)&n_entries, sizeof(n_entries));
183 if (in.fail() || n_entries < 1) {
184 printf("%s: no data in file %s\n", __func__, imatrix_file.c_str());
185 exit(1);
186 }
187 for (int i = 0; i < n_entries; ++i) {
188 int len; in.read((char *)&len, sizeof(len));
189 std::vector<char> name_as_vec(len+1);
190 in.read((char *)name_as_vec.data(), len);
191 if (in.fail()) {
192 printf("%s: failed reading name for entry %d from %s\n", __func__, i+1, imatrix_file.c_str());
193 exit(1);
194 }
195 name_as_vec[len] = 0;
196 std::string name{name_as_vec.data()};
197 auto & e = imatrix_data[name];
198 int ncall;
199 in.read((char *)&ncall, sizeof(ncall));
200 int nval;
201 in.read((char *)&nval, sizeof(nval));
202 if (in.fail() || nval < 1) {
203 printf("%s: failed reading number of values for entry %d\n", __func__, i);
204 imatrix_data = {};
205 exit(1);
206 }
207 e.resize(nval);
208 in.read((char *)e.data(), nval*sizeof(float));
209 if (in.fail()) {
210 printf("%s: failed reading data for entry %d\n", __func__, i);
211 imatrix_data = {};
212 exit(1);
213 }
214 if (ncall > 0) {
215 for (auto & v : e) {
216 v /= ncall;
217 }
218 }
219
220 if (getenv("LLAMA_TRACE")) {
221 printf("%s: loaded data (size = %6d, ncall = %6d) for '%s'\n", __func__, int(e.size()), ncall, name.c_str());
222 }
223 }
224
225 // latest legacy imatrix version contains the dataset filename at the end of the file
226 int m_last_call = 0;
227 if (in.peek() != EOF) {
228 in.read((char *)&m_last_call, sizeof(m_last_call));
229 int dataset_len;
230 in.read((char *)&dataset_len, sizeof(dataset_len));
231 std::vector<char> dataset_as_vec(dataset_len);
232 in.read(dataset_as_vec.data(), dataset_len);
233 imatrix_datasets.resize(1);
234 imatrix_datasets[0].assign(dataset_as_vec.begin(), dataset_as_vec.end());
235 printf("%s: imatrix dataset='%s'\n", __func__, imatrix_datasets[0].c_str());
236 }
237 printf("%s: loaded %d importance matrix entries from %s computed on %d chunks\n", __func__, int(imatrix_data.size()), imatrix_file.c_str(), m_last_call);
238 return m_last_call;
239}
240
241static int load_imatrix(const std::string & imatrix_file, std::vector<std::string> & imatrix_datasets, std::unordered_map<std::string, std::vector<float>> & imatrix_data) {
242
243 struct ggml_context * ctx = nullptr;
244 struct gguf_init_params meta_gguf_params = {
245 /* .no_alloc = */ false, // the data is needed
246 /* .ctx = */ &ctx,
247 };
248 struct gguf_context * ctx_gguf = gguf_init_from_file(imatrix_file.c_str(), meta_gguf_params);
249 if (!ctx_gguf) {
250 fprintf(stderr, "%s: imatrix file '%s' is using old format\n", __func__, imatrix_file.c_str());
251 return load_legacy_imatrix(imatrix_file, imatrix_datasets, imatrix_data);
252 }
253 const int32_t n_entries = gguf_get_n_tensors(ctx_gguf);
254 if (n_entries < 1) {
255 fprintf(stderr, "%s: no data in file %s\n", __func__, imatrix_file.c_str());
256 gguf_free(ctx_gguf);
257 ggml_free(ctx);
258 exit(1);
259 }
260
261 const int dataset_idx = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_DATASETS);
262 const int chunk_count_idx = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_CHUNK_COUNT);
263 const int chunk_size_idx = gguf_find_key(ctx_gguf, LLM_KV_IMATRIX_CHUNK_SIZE);
264 if (dataset_idx < 0 || chunk_count_idx < 0 || chunk_size_idx < 0) {
265 fprintf(stderr, "%s: missing imatrix metadata in file %s\n", __func__, imatrix_file.c_str());
266 gguf_free(ctx_gguf);
267 ggml_free(ctx);
268 exit(1);
269 }
270
271 const uint32_t chunk_size = gguf_get_val_u32(ctx_gguf, chunk_size_idx);
272
273 const std::string sums_suffix{ ".in_sum2" };
274 const std::string counts_suffix{ ".counts" };
275
276 // Using an ordered map to get a deterministic iteration order.
277 std::map<std::string, std::pair<struct ggml_tensor *, struct ggml_tensor *>> sums_counts_for;
278
279 for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {
280 std::string name = cur->name;
281
282 if (name.empty()) { continue; }
283
284 if (string_remove_suffix(name, sums_suffix)) {
285 // in_sum2
286 sums_counts_for[std::move(name)].first = cur;
287 } else if (string_remove_suffix(name, counts_suffix)) {
288 // counts
289 sums_counts_for[std::move(name)].second = cur;
290 } else {
291 // ignore other tensors
292 }
293 }
294
295 for (const auto & sc : sums_counts_for) {
296 const std::string & name = sc.first;
297 const struct ggml_tensor * sums = sc.second.first;
298 const struct ggml_tensor * counts = sc.second.second;
299
300 if (!sums || !counts) {
301 fprintf(stderr, "%s: mismatched sums and counts for %s\n", __func__, name.c_str());
302 gguf_free(ctx_gguf);
303 ggml_free(ctx);
304 exit(1);
305 }
306
307 const int64_t ne0 = sums->ne[0];
308 const int64_t ne1 = sums->ne[1];
309
310 auto & e = imatrix_data[name];
311 e.resize(ggml_nelements(sums));
312 float max_count = 0.0f;
313 for (int64_t j = 0; j < ne1; ++j) {
314 const float count = ((const float *) counts->data)[j];
315 if (count > 0.0f) {
316 for (int64_t i = 0; i < ne0; ++i) {
317 e[j*ne0 + i] = ((const float *) sums->data)[j*ne0 + i] / count;
318 }
319 } else {
320 // Partial imatrix data, this tensor never got any input during calibration
321 for (int64_t i = 0; i < ne0; ++i) {
322 e[j*ne0 + i] = 1;
323 }
324 }
325 if (count > max_count) {
326 max_count = count;
327 }
328 }
329 if (getenv("LLAMA_TRACE")) {
330 printf("%s: loaded data (size = %6d, n_tokens = %6d, n_chunks = %6d) for '%s'\n", __func__, int(e.size()), int(max_count), int(max_count / chunk_size), name.c_str());
331 }
332 }
333
334 int m_last_chunk = gguf_get_val_u32(ctx_gguf, chunk_count_idx);
335
336 int64_t n_datasets = gguf_get_arr_n(ctx_gguf, dataset_idx);
337 imatrix_datasets.reserve(n_datasets);
338 for (int64_t i = 0; i < n_datasets; ++i) {
339 imatrix_datasets.push_back(gguf_get_arr_str(ctx_gguf, dataset_idx, i));
340 }
341 printf("%s: imatrix datasets=['%s'", __func__, imatrix_datasets[0].c_str());
342 for (size_t i = 1; i < imatrix_datasets.size(); ++i) {
343 printf(", '%s'", imatrix_datasets[i].c_str());
344 }
345 printf("]\n");
346
347 printf("%s: loaded %d importance matrix entries from %s computed on %d chunks\n", __func__, int(imatrix_data.size()), imatrix_file.c_str(), m_last_chunk);
348
349 gguf_free(ctx_gguf);
350 ggml_free(ctx);
351
352 return m_last_chunk;
353}
354
355static int prepare_imatrix(const std::string & imatrix_file,
356 std::vector<std::string> & imatrix_dataset,
357 const std::vector<std::string> & included_weights,
358 const std::vector<std::string> & excluded_weights,
359 std::unordered_map<std::string, std::vector<float>> & imatrix_data) {
360 int m_last_call = -1;
361 if (!imatrix_file.empty()) {
362 m_last_call = load_imatrix(imatrix_file, imatrix_dataset, imatrix_data);
363 }
364 if (imatrix_data.empty()) {
365 return m_last_call;
366 }
367 if (!excluded_weights.empty()) {
368 for (const auto & name : excluded_weights) {
369 for (auto it = imatrix_data.begin(); it != imatrix_data.end();) {
370 auto pos = it->first.find(name);
371 if (pos != std::string::npos) {
372 it = imatrix_data.erase(it);
373 } else {
374 ++it;
375 }
376 }
377 }
378 }
379 if (!included_weights.empty()) {
380 std::unordered_map<std::string, std::vector<float>> tmp;
381 for (const auto & name : included_weights) {
382 for (auto & e : imatrix_data) {
383 auto pos = e.first.find(name);
384 if (pos != std::string::npos) {
385 tmp.emplace(std::move(e));
386 }
387 }
388 }
389 imatrix_data = std::move(tmp);
390 }
391 if (!imatrix_data.empty()) {
392 printf("%s: have %d importance matrix entries\n", __func__, int(imatrix_data.size()));
393 }
394 return m_last_call;
395}
396
397static ggml_type parse_ggml_type(const char * arg) {
398 for (int i = 0; i < GGML_TYPE_COUNT; ++i) {
399 auto type = (ggml_type)i;
400 const auto * name = ggml_type_name(type);
401 if (name && striequals(name, arg)) {
402 return type;
403 }
404 }
405 fprintf(stderr, "\n%s: invalid ggml_type '%s'\n\n", __func__, arg);
406 return GGML_TYPE_COUNT;
407}
408
409static bool parse_tensor_type(const char * data, std::vector<tensor_quantization> & tensor_type) {
410 const char * sep = strchr(data, '=');
411 if (sep == nullptr) {
412 printf("\n%s: malformed tensor type '%s'\n\n", __func__, data);
413 return false;
414 }
415
416 const size_t tn_len = sep - data;
417 if (tn_len == 0) {
418 printf("\n%s: missing tensor name\n\n", __func__);
419 return false;
420 }
421 if (const size_t qt_len = strlen(sep); qt_len == 1) {
422 printf("\n%s: missing quantization type\n\n", __func__);
423 return false;
424 }
425
426 std::string tn(data, tn_len);
427 std::transform(tn.begin(), tn.end(), tn.begin(), tolower);
428 sep++;
429 tensor_quantization tqz;
430 tqz.name = tn;
431 tqz.quant = parse_ggml_type(sep);
432 tensor_type.emplace_back(std::move(tqz));
433 if (tqz.quant == GGML_TYPE_COUNT) {
434 printf("\n%s: invalid quantization type '%s'\n\n", __func__, sep);
435 return false;
436 }
437
438 return true;
439}
440
441static bool parse_tensor_type_file(const char * filename, std::vector<tensor_quantization> & tensor_type) {
442 std::ifstream file(filename);
443 if (!file) {
444 printf("\n%s: failed to open file '%s': %s\n\n", __func__, filename, std::strerror(errno));
445 return false;
446 }
447
448 std::string arg;
449 while (file >> arg) {
450 if (!parse_tensor_type(arg.c_str(), tensor_type)) {
451 return false;
452 }
453 }
454
455 return true;
456}
457
458static bool parse_layer_prune(const char * data, std::vector<int> & prune_layers) {
459 if (!data) {
460 printf("\n%s: no layer pruning ids provided\n\n", __func__);
461 return false;
462 }
463
464 const auto block_ids = string_split<std::string>(data, ',');
465 for (const auto & block_id : block_ids) {
466 int id;
467 try {
468 id = std::stoi(block_id);
469 } catch (...) {
470 id = -1;
471 }
472 if (id < 0) {
473 printf("\n%s: invalid layer id '%s'\n\n", __func__, block_id.c_str());
474 return false;
475 }
476 prune_layers.emplace_back(id);
477 }
478
479 sort(prune_layers.begin(), prune_layers.end());
480 prune_layers.erase(std::unique(prune_layers.begin(), prune_layers.end()), prune_layers.end());
481 return true;
482}
483
484int main(int argc, char ** argv) {
485 if (argc < 3) {
486 usage(argv[0]);
487 }
488
489 llama_model_quantize_params params = llama_model_quantize_default_params();
490
491 int arg_idx = 1;
492 std::string imatrix_file;
493 std::vector<std::string> included_weights, excluded_weights;
494 std::vector<llama_model_kv_override> kv_overrides;
495 std::vector<tensor_quantization> tensor_types;
496 std::vector<int> prune_layers;
497
498 for (; arg_idx < argc && strncmp(argv[arg_idx], "--", 2) == 0; arg_idx++) {
499 if (strcmp(argv[arg_idx], "--leave-output-tensor") == 0) {
500 params.quantize_output_tensor = false;
501 } else if (strcmp(argv[arg_idx], "--output-tensor-type") == 0) {
502 if (arg_idx < argc-1) {
503 params.output_tensor_type = parse_ggml_type(argv[++arg_idx]);
504 if (params.output_tensor_type == GGML_TYPE_COUNT) {
505 usage(argv[0]);
506 }
507 } else {
508 usage(argv[0]);
509 }
510 } else if (strcmp(argv[arg_idx], "--token-embedding-type") == 0) {
511 if (arg_idx < argc-1) {
512 params.token_embedding_type = parse_ggml_type(argv[++arg_idx]);
513 if (params.token_embedding_type == GGML_TYPE_COUNT) {
514 usage(argv[0]);
515 }
516 } else {
517 usage(argv[0]);
518 }
519 } else if (strcmp(argv[arg_idx], "--tensor-type") == 0) {
520 if (arg_idx == argc-1 || !parse_tensor_type(argv[++arg_idx], tensor_types)) {
521 usage(argv[0]);
522 }
523 } else if (strcmp(argv[arg_idx], "--tensor-type-file") == 0) {
524 if (arg_idx == argc-1 || !parse_tensor_type_file(argv[++arg_idx], tensor_types)) {
525 usage(argv[0]);
526 }
527 } else if (strcmp(argv[arg_idx], "--prune-layers") == 0) {
528 if (arg_idx == argc-1 || !parse_layer_prune(argv[++arg_idx], prune_layers)) {
529 usage(argv[0]);
530 }
531 } else if (strcmp(argv[arg_idx], "--override-kv") == 0) {
532 if (arg_idx == argc-1 || !string_parse_kv_override(argv[++arg_idx], kv_overrides)) {
533 usage(argv[0]);
534 }
535 } else if (strcmp(argv[arg_idx], "--allow-requantize") == 0) {
536 params.allow_requantize = true;
537 } else if (strcmp(argv[arg_idx], "--pure") == 0) {
538 params.pure = true;
539 } else if (strcmp(argv[arg_idx], "--imatrix") == 0) {
540 if (arg_idx < argc-1) {
541 imatrix_file = argv[++arg_idx];
542 } else {
543 usage(argv[0]);
544 }
545 } else if (strcmp(argv[arg_idx], "--include-weights") == 0) {
546 if (arg_idx < argc-1) {
547 included_weights.emplace_back(argv[++arg_idx]);
548 } else {
549 usage(argv[0]);
550 }
551 } else if (strcmp(argv[arg_idx], "--exclude-weights") == 0) {
552 if (arg_idx < argc-1) {
553 excluded_weights.emplace_back(argv[++arg_idx]);
554 } else {
555 usage(argv[0]);
556 }
557 } else if (strcmp(argv[arg_idx], "--keep-split") == 0) {
558 params.keep_split = true;
559 } else {
560 usage(argv[0]);
561 }
562 }
563
564 if (argc - arg_idx < 2) {
565 printf("%s: bad arguments\n", argv[0]);
566 usage(argv[0]);
567 }
568 if (!included_weights.empty() && !excluded_weights.empty()) {
569 usage(argv[0]);
570 }
571
572 std::vector<std::string> imatrix_datasets;
573 std::unordered_map<std::string, std::vector<float>> imatrix_data;
574 int m_last_call = prepare_imatrix(imatrix_file, imatrix_datasets, included_weights, excluded_weights, imatrix_data);
575 if (!imatrix_data.empty()) {
576 params.imatrix = &imatrix_data;
577 {
578 llama_model_kv_override kvo;
579 std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_FILE);
580 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR;
581 strncpy(kvo.val_str, imatrix_file.c_str(), 127);
582 kvo.val_str[127] = '\0';
583 kv_overrides.emplace_back(std::move(kvo));
584 }
585 if (!imatrix_datasets.empty()) {
586 llama_model_kv_override kvo;
587 // TODO: list multiple datasets when there are more than one
588 std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_DATASET);
589 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR;
590 strncpy(kvo.val_str, imatrix_datasets[0].c_str(), 127);
591 kvo.val_str[127] = '\0';
592 kv_overrides.emplace_back(std::move(kvo));
593 }
594
595 {
596 llama_model_kv_override kvo;
597 std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_N_ENTRIES);
598 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;
599 kvo.val_i64 = imatrix_data.size();
600 kv_overrides.emplace_back(std::move(kvo));
601 }
602
603 if (m_last_call > 0) {
604 llama_model_kv_override kvo;
605 std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_N_CHUNKS);
606 kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;
607 kvo.val_i64 = m_last_call;
608 kv_overrides.emplace_back(std::move(kvo));
609 }
610 }
611 if (!kv_overrides.empty()) {
612 kv_overrides.emplace_back();
613 kv_overrides.back().key[0] = 0;
614 params.kv_overrides = &kv_overrides;
615 }
616 if (!tensor_types.empty()) {
617 params.tensor_types = &tensor_types;
618 }
619 if (!prune_layers.empty()) {
620 params.prune_layers = &prune_layers;
621 }
622
623 llama_backend_init();
624
625 // parse command line arguments
626 const std::string fname_inp = argv[arg_idx];
627 arg_idx++;
628 std::string fname_out;
629
630 std::string ftype_str;
631 std::string suffix = ".gguf";
632 if (try_parse_ftype(argv[arg_idx], params.ftype, ftype_str)) {
633 std::string fpath;
634 const size_t pos = fname_inp.find_last_of("/\\");
635 if (pos != std::string::npos) {
636 fpath = fname_inp.substr(0, pos + 1);
637 }
638
639 // export as [inp path]/ggml-model-[ftype]. Only add extension if there is no splitting
640 fname_out = fpath + "ggml-model-" + ftype_str;
641 if (!params.keep_split) {
642 fname_out += suffix;
643 }
644 arg_idx++;
645 if (ftype_str == "COPY") {
646 params.only_copy = true;
647 }
648 } else {
649 fname_out = argv[arg_idx];
650 if (params.keep_split && fname_out.find(suffix) != std::string::npos) {
651 fname_out = fname_out.substr(0, fname_out.length() - suffix.length());
652 }
653 arg_idx++;
654
655 if (argc <= arg_idx) {
656 fprintf(stderr, "%s: missing ftype\n", __func__);
657 return 1;
658 }
659 if (!try_parse_ftype(argv[arg_idx], params.ftype, ftype_str)) {
660 fprintf(stderr, "%s: invalid ftype '%s'\n", __func__, argv[arg_idx]);
661 return 1;
662 }
663 if (ftype_str == "COPY") {
664 params.only_copy = true;
665 }
666 arg_idx++;
667 }
668
669 // parse nthreads
670 if (argc > arg_idx) {
671 try {
672 params.nthread = std::stoi(argv[arg_idx]);
673 }
674 catch (const std::exception & e) {
675 fprintf(stderr, "%s: invalid nthread '%s' (%s)\n", __func__, argv[arg_idx], e.what());
676 return 1;
677 }
678 }
679
680 if ((params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS ||
681 params.ftype == LLAMA_FTYPE_MOSTLY_IQ2_S ||
682 params.ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S ||
683 params.ftype == LLAMA_FTYPE_MOSTLY_IQ1_S ||
684 params.ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) && imatrix_data.empty()) {
685 fprintf(stderr, "\n==========================================================================================================\n");
686 fprintf(stderr, "Please do not use IQ1_S, IQ1_M, IQ2_S, IQ2_XXS, IQ2_XS or Q2_K_S quantization without an importance matrix\n");
687 fprintf(stderr, "==========================================================================================================\n\n\n");
688 return 1;
689 }
690
691 if (std::error_code ec; std::filesystem::equivalent(fname_inp, fname_out, ec)) {
692 fprintf(stderr, "%s: error: input and output files are the same: '%s'\n", __func__, fname_inp.c_str());
693 return 1;
694 }
695
696 print_build_info();
697
698 fprintf(stderr, "%s: quantizing '%s' to '%s' as %s", __func__, fname_inp.c_str(), fname_out.c_str(), ftype_str.c_str());
699 if (params.nthread > 0) {
700 fprintf(stderr, " using %d threads", params.nthread);
701 }
702 fprintf(stderr, "\n");
703
704 const int64_t t_main_start_us = llama_time_us();
705
706 int64_t t_quantize_us = 0;
707
708 // load the model
709 {
710 const int64_t t_start_us = llama_time_us();
711
712 if (llama_model_quantize(fname_inp.c_str(), fname_out.c_str(), ¶ms)) {
713 fprintf(stderr, "%s: failed to quantize model from '%s'\n", __func__, fname_inp.c_str());
714 return 1;
715 }
716
717 t_quantize_us = llama_time_us() - t_start_us;
718 }
719
720 // report timing
721 {
722 const int64_t t_main_end_us = llama_time_us();
723
724 printf("\n");
725 printf("%s: quantize time = %8.2f ms\n", __func__, t_quantize_us/1000.0);
726 printf("%s: total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0);
727 }
728
729 llama_backend_free();
730
731 return 0;
732}
733