1#include "ggml.h"
2#include "ggml-backend.h"
3#include "ggml-impl.h"
4#include "gguf.h"
5
6#include <cinttypes>
7#include <cstddef>
8#include <cstdint>
9#include <cstdio>
10#include <cstdlib>
11#include <cstring>
12#include <map>
13#include <new>
14#include <stdexcept>
15#include <string>
16#include <vector>
17
18template <typename T>
19struct type_to_gguf_type;
20
21template <>
22struct type_to_gguf_type<uint8_t> {
23 static constexpr enum gguf_type value = GGUF_TYPE_UINT8;
24};
25
26template <>
27struct type_to_gguf_type<int8_t> {
28 static constexpr enum gguf_type value = GGUF_TYPE_INT8;
29};
30
31template <>
32struct type_to_gguf_type<uint16_t> {
33 static constexpr enum gguf_type value = GGUF_TYPE_UINT16;
34};
35
36template <>
37struct type_to_gguf_type<int16_t> {
38 static constexpr enum gguf_type value = GGUF_TYPE_INT16;
39};
40
41template <>
42struct type_to_gguf_type<uint32_t> {
43 static constexpr enum gguf_type value = GGUF_TYPE_UINT32;
44};
45
46template <>
47struct type_to_gguf_type<int32_t> {
48 static constexpr enum gguf_type value = GGUF_TYPE_INT32;
49};
50
51template <>
52struct type_to_gguf_type<float> {
53 static constexpr enum gguf_type value = GGUF_TYPE_FLOAT32;
54};
55
56template <>
57struct type_to_gguf_type<bool> {
58 static constexpr enum gguf_type value = GGUF_TYPE_BOOL;
59};
60
61template <>
62struct type_to_gguf_type<std::string> {
63 static constexpr enum gguf_type value = GGUF_TYPE_STRING;
64};
65
66template <>
67struct type_to_gguf_type<uint64_t> {
68 static constexpr enum gguf_type value = GGUF_TYPE_UINT64;
69};
70
71template <>
72struct type_to_gguf_type<int64_t> {
73 static constexpr enum gguf_type value = GGUF_TYPE_INT64;
74};
75
76template <>
77struct type_to_gguf_type<double> {
78 static constexpr enum gguf_type value = GGUF_TYPE_FLOAT64;
79};
80
81static const std::map<gguf_type, size_t> GGUF_TYPE_SIZE = {
82 {GGUF_TYPE_UINT8, sizeof(uint8_t)},
83 {GGUF_TYPE_INT8, sizeof(int8_t)},
84 {GGUF_TYPE_UINT16, sizeof(uint16_t)},
85 {GGUF_TYPE_INT16, sizeof(int16_t)},
86 {GGUF_TYPE_UINT32, sizeof(uint32_t)},
87 {GGUF_TYPE_INT32, sizeof(int32_t)},
88 {GGUF_TYPE_FLOAT32, sizeof(float)},
89 {GGUF_TYPE_BOOL, sizeof(int8_t)},
90 {GGUF_TYPE_STRING, 0}, // undefined
91 {GGUF_TYPE_ARRAY, 0}, // undefined
92 {GGUF_TYPE_UINT64, sizeof(uint64_t)},
93 {GGUF_TYPE_INT64, sizeof(int64_t)},
94 {GGUF_TYPE_FLOAT64, sizeof(double)},
95};
96static_assert(GGUF_TYPE_COUNT == 13, "GGUF_TYPE_COUNT != 13");
97
98static const std::map<gguf_type, const char *> GGUF_TYPE_NAME = {
99 {GGUF_TYPE_UINT8, "u8"},
100 {GGUF_TYPE_INT8, "i8"},
101 {GGUF_TYPE_UINT16, "u16"},
102 {GGUF_TYPE_INT16, "i16"},
103 {GGUF_TYPE_UINT32, "u32"},
104 {GGUF_TYPE_INT32, "i32"},
105 {GGUF_TYPE_FLOAT32, "f32"},
106 {GGUF_TYPE_BOOL, "bool"},
107 {GGUF_TYPE_STRING, "str"},
108 {GGUF_TYPE_ARRAY, "arr"},
109 {GGUF_TYPE_UINT64, "u64"},
110 {GGUF_TYPE_INT64, "i64"},
111 {GGUF_TYPE_FLOAT64, "f64"},
112};
113static_assert(GGUF_TYPE_COUNT == 13, "GGUF_TYPE_COUNT != 13");
114
115size_t gguf_type_size(enum gguf_type type) {
116 auto it = GGUF_TYPE_SIZE.find(type);
117 return it == GGUF_TYPE_SIZE.end() ? 0 : it->second;
118}
119
120struct gguf_kv {
121 std::string key;
122
123 bool is_array;
124 enum gguf_type type;
125
126 std::vector<int8_t> data;
127 std::vector<std::string> data_string;
128
129 template <typename T>
130 gguf_kv(const std::string & key, const T value)
131 : key(key), is_array(false), type(type_to_gguf_type<T>::value) {
132 GGML_ASSERT(!key.empty());
133 data.resize(sizeof(T));
134 memcpy(data.data(), &value, sizeof(T));
135 }
136
137 template <typename T>
138 gguf_kv(const std::string & key, const std::vector<T> & value)
139 : key(key), is_array(true), type(type_to_gguf_type<T>::value) {
140 GGML_ASSERT(!key.empty());
141 data.resize(value.size()*sizeof(T));
142 for (size_t i = 0; i < value.size(); ++i) {
143 const T tmp = value[i];
144 memcpy(data.data() + i*sizeof(T), &tmp, sizeof(T));
145 }
146 }
147
148 gguf_kv(const std::string & key, const std::string & value)
149 : key(key), is_array(false), type(GGUF_TYPE_STRING) {
150 GGML_ASSERT(!key.empty());
151 data_string.push_back(value);
152 }
153
154 gguf_kv(const std::string & key, const std::vector<std::string> & value)
155 : key(key), is_array(true), type(GGUF_TYPE_STRING) {
156 GGML_ASSERT(!key.empty());
157 data_string = value;
158 }
159
160 const std::string & get_key() const {
161 return key;
162 }
163
164 const enum gguf_type & get_type() const {
165 return type;
166 }
167
168 size_t get_ne() const {
169 if (type == GGUF_TYPE_STRING) {
170 const size_t ne = data_string.size();
171 GGML_ASSERT(is_array || ne == 1);
172 return ne;
173 }
174 const size_t type_size = gguf_type_size(type);
175 GGML_ASSERT(data.size() % type_size == 0);
176 const size_t ne = data.size() / type_size;
177 GGML_ASSERT(is_array || ne == 1);
178 return ne;
179 }
180
181 template <typename T>
182 const T & get_val(const size_t i = 0) const {
183 GGML_ASSERT(type_to_gguf_type<T>::value == type);
184 if constexpr (std::is_same<T, std::string>::value) {
185 GGML_ASSERT(data_string.size() >= i+1);
186 return data_string[i];
187 }
188 const size_t type_size = gguf_type_size(type);
189 GGML_ASSERT(data.size() % type_size == 0);
190 GGML_ASSERT(data.size() >= (i+1)*type_size);
191 return reinterpret_cast<const T *>(data.data())[i];
192 }
193
194 void cast(const enum gguf_type new_type) {
195 const size_t new_type_size = gguf_type_size(new_type);
196 GGML_ASSERT(data.size() % new_type_size == 0);
197 type = new_type;
198 }
199};
200
201struct gguf_tensor_info {
202 struct ggml_tensor t; // for holding the equivalent info
203 uint64_t offset; // offset from start of `data`, must be a multiple of `ALIGNMENT`
204};
205
206struct gguf_context {
207 uint32_t version = GGUF_VERSION;
208
209 std::vector<struct gguf_kv> kv;
210 std::vector<struct gguf_tensor_info> info;
211
212 size_t alignment = GGUF_DEFAULT_ALIGNMENT;
213 size_t offset = 0; // offset of `data` from beginning of file
214 size_t size = 0; // size of `data` in bytes
215
216 void * data = nullptr;
217};
218
219struct gguf_reader {
220 FILE * file;
221
222 gguf_reader(FILE * file) : file(file) {}
223
224 template <typename T>
225 bool read(T & dst) const {
226 return fread(&dst, 1, sizeof(dst), file) == sizeof(dst);
227 }
228
229 template <typename T>
230 bool read(std::vector<T> & dst, const size_t n) const {
231 dst.resize(n);
232 for (size_t i = 0; i < dst.size(); ++i) {
233 if constexpr (std::is_same<T, bool>::value) {
234 bool tmp;
235 if (!read(tmp)) {
236 return false;
237 }
238 dst[i] = tmp;
239 } else {
240 if (!read(dst[i])) {
241 return false;
242 }
243 }
244 }
245 return true;
246 }
247
248 bool read(bool & dst) const {
249 int8_t tmp = -1;
250 if (!read(tmp)) {
251 return false;
252 }
253 dst = tmp != 0;
254 return true;
255 }
256
257 bool read(enum ggml_type & dst) const {
258 int32_t tmp = -1;
259 if (!read(tmp)) {
260 return false;
261 }
262 dst = ggml_type(tmp);
263 return true;
264 }
265
266 bool read(enum gguf_type & dst) const {
267 int32_t tmp = -1;
268 if (!read(tmp)) {
269 return false;
270 }
271 dst = gguf_type(tmp);
272 return true;
273 }
274
275 bool read(std::string & dst) const {
276 uint64_t size = 0;
277 if (!read(size)) {
278 return false;
279 }
280 dst.resize(size);
281 return fread(dst.data(), 1, dst.length(), file) == dst.length();
282 }
283
284 bool read(void * dst, const size_t size) const {
285 return fread(dst, 1, size, file) == size;
286 }
287};
288
289struct gguf_context * gguf_init_empty(void) {
290 return new gguf_context;
291}
292
293template<typename T>
294bool gguf_read_emplace_helper(const struct gguf_reader & gr, std::vector<struct gguf_kv> & kv, const std::string & key, const bool is_array, const size_t n) {
295 if (is_array) {
296 std::vector<T> value;
297 try {
298 if (!gr.read(value, n)) {
299 return false;
300 }
301 } catch (std::length_error &) {
302 GGML_LOG_ERROR("%s: encountered length_error while reading value for key '%s'\n", __func__, key.c_str());
303 return false;
304 } catch (std::bad_alloc &) {
305 GGML_LOG_ERROR("%s: encountered bad_alloc error while reading value for key '%s'\n", __func__, key.c_str());
306 return false;
307 }
308 kv.emplace_back(key, value);
309 } else {
310 T value;
311 if (!gr.read(value)) {
312 return false;
313 }
314 kv.emplace_back(key, value);
315 }
316 return true;
317}
318
319struct gguf_context * gguf_init_from_file_impl(FILE * file, struct gguf_init_params params) {
320 const struct gguf_reader gr(file);
321 struct gguf_context * ctx = new gguf_context;
322
323 bool ok = true;
324
325 // file magic
326 {
327 std::vector<char> magic;
328 ok = ok && gr.read(magic, 4);
329
330 if (!ok) {
331 GGML_LOG_ERROR("%s: failed to read magic\n", __func__);
332 gguf_free(ctx);
333 return nullptr;
334 }
335
336 for (uint32_t i = 0; i < magic.size(); i++) {
337 if (magic[i] != GGUF_MAGIC[i]) {
338 char c0 = isprint(magic[0]) ? magic[0] : '?';
339 char c1 = isprint(magic[1]) ? magic[1] : '?';
340 char c2 = isprint(magic[2]) ? magic[2] : '?';
341 char c3 = isprint(magic[3]) ? magic[3] : '?';
342 GGML_LOG_ERROR("%s: invalid magic characters: '%c%c%c%c', expected 'GGUF'\n", __func__, c0, c1, c2, c3);
343 gguf_free(ctx);
344 return nullptr;
345 }
346 }
347 }
348
349 // header
350 int64_t n_kv = 0;
351 int64_t n_tensors = 0;
352
353 if (ok && gr.read(ctx->version)) {
354 if (ok && ctx->version == 0) {
355 GGML_LOG_ERROR("%s: bad GGUF version: %" PRIu32 "\n", __func__, ctx->version);
356 ok = false;
357 }
358
359 /*
360 * bit layout is different when reading non-native endian models.
361 * assuming that the GGUF version is 3, the non-native endian model
362 * would read it as 0x30000000. we can use the AND operation against
363 * the last 4 hexadecimal digits to check if the model is the same
364 * endianness as the host system.
365 */
366 if (ok && (ctx->version & 0x0000FFFF) == 0x00000000) {
367 GGML_LOG_ERROR("%s: failed to load model: this GGUF file version %" PRIu32 " is extremely large, is there a mismatch between the host and model endianness?\n", __func__, ctx->version);
368 ok = false;
369 }
370
371 if (ok && ctx->version == 1) {
372 GGML_LOG_ERROR("%s: GGUFv1 is no longer supported, please use a more up-to-date version\n", __func__);
373 ok = false;
374 }
375 if (ok && ctx->version > GGUF_VERSION) {
376 GGML_LOG_ERROR("%s: this GGUF file is version %" PRIu32 " but this software only supports up to version %d\n",
377 __func__, ctx->version, GGUF_VERSION);
378 ok = false;
379 }
380 } else {
381 ok = false;
382 }
383
384 if (ok && gr.read(n_tensors)) {
385 static_assert(sizeof(size_t) <= 8 && sizeof(gguf_tensor_info) >= 2, "int64_t insufficient for indexing");
386 if (n_tensors < 0 || n_tensors > int64_t(SIZE_MAX/sizeof(gguf_tensor_info))) {
387 GGML_LOG_ERROR("%s: number of tensors is %" PRIi64 " but must be in [0, %zu]\n",
388 __func__, n_tensors, SIZE_MAX/sizeof(gguf_tensor_info));
389 ok = false;
390 }
391 } else {
392 ok = false;
393 }
394
395 if (ok && gr.read(n_kv)) {
396 static_assert(sizeof(size_t) <= 8 && sizeof(gguf_tensor_info) >= 2, "int64_t insufficient for indexing");
397 if (n_kv < 0 || n_kv > int64_t(SIZE_MAX/sizeof(gguf_kv))) {
398 GGML_LOG_ERROR("%s: number of key value pairs is %" PRIi64 " but must be in [0, %zu]\n",
399 __func__, n_kv, SIZE_MAX/sizeof(gguf_kv));
400 ok = false;
401 }
402 } else {
403 ok = false;
404 }
405
406 if (!ok) {
407 GGML_LOG_ERROR("%s: failed to read header\n", __func__);
408 gguf_free(ctx);
409 return nullptr;
410 }
411
412 // KV pairs
413 {
414 for (int64_t i = 0; ok && i < n_kv; ++i) {
415 std::string key;
416 gguf_type type = gguf_type(-1);
417 bool is_array = false;
418 uint64_t n = 1;
419
420 try {
421 ok = ok && gr.read(key);
422 } catch (std::length_error &) {
423 GGML_LOG_ERROR("%s: encountered length_error while reading key %" PRIi64 "\n", __func__, i);
424 ok = false;
425 } catch (std::bad_alloc &) {
426 GGML_LOG_ERROR("%s: encountered bad_alloc error while reading key %" PRIi64 "\n", __func__, i);
427 ok = false;
428 }
429 for (size_t j = 0; ok && j < ctx->kv.size(); ++j) {
430 if (key == ctx->kv[j].key) {
431 GGML_LOG_ERROR("%s: duplicate key '%s' for tensors %zu and %" PRIi64 " \n", __func__, key.c_str(), j, i);
432 ok = false;
433 }
434 }
435 if (!ok) {
436 break;
437 }
438
439 ok = ok && gr.read(type);
440 if (type == GGUF_TYPE_ARRAY) {
441 is_array = true;
442 ok = ok && gr.read(type);
443 ok = ok && gr.read(n);
444 }
445 if (!ok) {
446 break;
447 }
448
449 switch (type) {
450 case GGUF_TYPE_UINT8: ok = ok && gguf_read_emplace_helper<uint8_t> (gr, ctx->kv, key, is_array, n); break;
451 case GGUF_TYPE_INT8: ok = ok && gguf_read_emplace_helper<int8_t> (gr, ctx->kv, key, is_array, n); break;
452 case GGUF_TYPE_UINT16: ok = ok && gguf_read_emplace_helper<uint16_t> (gr, ctx->kv, key, is_array, n); break;
453 case GGUF_TYPE_INT16: ok = ok && gguf_read_emplace_helper<int16_t> (gr, ctx->kv, key, is_array, n); break;
454 case GGUF_TYPE_UINT32: ok = ok && gguf_read_emplace_helper<uint32_t> (gr, ctx->kv, key, is_array, n); break;
455 case GGUF_TYPE_INT32: ok = ok && gguf_read_emplace_helper<int32_t> (gr, ctx->kv, key, is_array, n); break;
456 case GGUF_TYPE_FLOAT32: ok = ok && gguf_read_emplace_helper<float> (gr, ctx->kv, key, is_array, n); break;
457 case GGUF_TYPE_BOOL: ok = ok && gguf_read_emplace_helper<bool> (gr, ctx->kv, key, is_array, n); break;
458 case GGUF_TYPE_STRING: ok = ok && gguf_read_emplace_helper<std::string>(gr, ctx->kv, key, is_array, n); break;
459 case GGUF_TYPE_UINT64: ok = ok && gguf_read_emplace_helper<uint64_t> (gr, ctx->kv, key, is_array, n); break;
460 case GGUF_TYPE_INT64: ok = ok && gguf_read_emplace_helper<int64_t> (gr, ctx->kv, key, is_array, n); break;
461 case GGUF_TYPE_FLOAT64: ok = ok && gguf_read_emplace_helper<double> (gr, ctx->kv, key, is_array, n); break;
462 case GGUF_TYPE_ARRAY:
463 default:
464 {
465 GGML_LOG_ERROR("%s: key '%s' has invalid GGUF type %d\n", __func__, key.c_str(), type);
466 ok = false;
467 } break;
468 }
469 }
470
471 if (!ok) {
472 GGML_LOG_ERROR("%s: failed to read key-value pairs\n", __func__);
473 gguf_free(ctx);
474 return nullptr;
475 }
476 GGML_ASSERT(int64_t(ctx->kv.size()) == n_kv);
477
478 const int alignment_idx = gguf_find_key(ctx, GGUF_KEY_GENERAL_ALIGNMENT);
479 ctx->alignment = alignment_idx == -1 ? GGUF_DEFAULT_ALIGNMENT : gguf_get_val_u32(ctx, alignment_idx);
480
481 if (ctx->alignment == 0 || (ctx->alignment & (ctx->alignment - 1)) != 0) {
482 GGML_LOG_ERROR("%s: alignment %zu is not a power of 2\n", __func__, ctx->alignment);
483 gguf_free(ctx);
484 return nullptr;
485 }
486 }
487
488 // read the tensor info
489 for (int64_t i = 0; ok && i < n_tensors; ++i) {
490 struct gguf_tensor_info info;
491
492 // tensor name
493 {
494 std::string name;
495 try {
496 ok = ok && gr.read(name);
497 } catch (std::length_error &) {
498 GGML_LOG_ERROR("%s: encountered length_error while reading tensor name %" PRIi64 "\n", __func__, i);
499 ok = false;
500 } catch (std::bad_alloc &) {
501 GGML_LOG_ERROR("%s: encountered bad_alloc error while reading tensor name %" PRIi64 "\n", __func__, i);
502 ok = false;
503 }
504 if (name.length() >= GGML_MAX_NAME) {
505 GGML_LOG_ERROR("%s: tensor name %" PRIi64 " is too long: %zu >= %d\n", __func__, i, name.length(), GGML_MAX_NAME);
506 ok = false;
507 break;
508 }
509 ggml_set_name(&info.t, name.c_str());
510
511 // make sure there are no duplicate tensor names
512 for (int64_t j = 0; ok && j < i; ++j) {
513 if (strcmp(info.t.name, ctx->info[j].t.name) == 0) {
514 GGML_LOG_ERROR("%s: duplicate tensor name '%s' for tensors %" PRIi64 " and %" PRIi64 "\n", __func__, info.t.name, j, i);
515 ok = false;
516 break;
517 }
518 }
519 }
520 if (!ok) {
521 break;
522 }
523
524 // tensor shape
525 {
526 uint32_t n_dims = 0;
527 ok = ok && gr.read(n_dims);
528 if (n_dims > GGML_MAX_DIMS) {
529 GGML_LOG_ERROR("%s: tensor '%s' has invalid number of dimensions: %" PRIu32 " > %" PRIu32 "\n",
530 __func__, info.t.name, n_dims, GGML_MAX_DIMS);
531 ok = false;
532 break;
533 }
534 for (uint32_t j = 0; ok && j < GGML_MAX_DIMS; ++j) {
535 info.t.ne[j] = 1;
536 if (j < n_dims) {
537 ok = ok && gr.read(info.t.ne[j]);
538 }
539
540 // check that all ne are non-negative
541 if (info.t.ne[j] < 0) {
542 GGML_LOG_ERROR("%s: tensor '%s' dimension %" PRIu32 " has invalid number of elements: %" PRIi64 " < 0\n",
543 __func__, info.t.name, j, info.t.ne[j]);
544 ok = false;
545 break;
546 }
547 }
548
549 // check that the total number of elements is representable
550 if (ok && ((INT64_MAX/info.t.ne[1] <= info.t.ne[0]) ||
551 (INT64_MAX/info.t.ne[2] <= info.t.ne[0]*info.t.ne[1]) ||
552 (INT64_MAX/info.t.ne[3] <= info.t.ne[0]*info.t.ne[1]*info.t.ne[2]))) {
553
554 GGML_LOG_ERROR("%s: total number of elements in tensor '%s' with shape "
555 "(%" PRIi64 ", %" PRIi64 ", %" PRIi64 ", %" PRIi64 ") is >= %" PRIi64 "\n",
556 __func__, info.t.name, info.t.ne[0], info.t.ne[1], info.t.ne[2], info.t.ne[3], INT64_MAX);
557 ok = false;
558 break;
559 }
560 }
561 if (!ok) {
562 break;
563 }
564
565 // tensor type
566 {
567 ok = ok && gr.read(info.t.type);
568
569 // check that tensor type is within defined range
570 if (info.t.type < 0 || info.t.type >= GGML_TYPE_COUNT) {
571 GGML_LOG_ERROR("%s: tensor '%s' has invalid ggml type %d (%s)\n",
572 __func__, info.t.name, info.t.type, ggml_type_name(info.t.type));
573 ok = false;
574 break;
575 }
576 const size_t type_size = ggml_type_size(info.t.type);
577 const int64_t blck_size = ggml_blck_size(info.t.type);
578
579 // check that row size is divisible by block size
580 if (blck_size == 0 || info.t.ne[0] % blck_size != 0) {
581 GGML_LOG_ERROR("%s: tensor '%s' of type %d (%s) has %" PRId64 " elements per row, "
582 "not a multiple of block size (%" PRId64 ")\n",
583 __func__, info.t.name, (int) info.t.type, ggml_type_name(info.t.type), info.t.ne[0], blck_size);
584 ok = false;
585 break;
586 }
587
588 // check that the size of the tensor in bytes is representable
589 if (ok && uint64_t(ggml_nelements(&info.t)/ggml_blck_size(info.t.type)) > SIZE_MAX/ggml_type_size(info.t.type)) {
590 GGML_LOG_ERROR("%s: tensor '%s' with shape (%" PRIi64 ", %" PRIi64 ", %" PRIi64 ", %" PRIi64 ") has a size in bytes > %zu\n",
591 __func__, info.t.name, info.t.ne[0], info.t.ne[1], info.t.ne[2], info.t.ne[3], SIZE_MAX);
592 ok = false;
593 break;
594 }
595
596 // calculate byte offsets given the tensor shape and type
597 info.t.nb[0] = type_size;
598 info.t.nb[1] = info.t.nb[0]*(info.t.ne[0]/blck_size);
599 for (int j = 2; j < GGML_MAX_DIMS; ++j) {
600 info.t.nb[j] = info.t.nb[j - 1]*info.t.ne[j - 1];
601 }
602 }
603 if (!ok) {
604 break;
605 }
606
607 // tensor data offset within buffer
608 ok = ok && gr.read(info.offset);
609
610 ctx->info.push_back(info);
611 }
612
613 if (!ok) {
614 GGML_LOG_ERROR("%s: failed to read tensor info\n", __func__);
615 gguf_free(ctx);
616 return nullptr;
617 }
618 GGML_ASSERT(int64_t(ctx->info.size()) == n_tensors);
619
620 // we require the data section to be aligned, so take into account any padding
621 if (fseek(file, GGML_PAD(ftell(file), ctx->alignment), SEEK_SET) != 0) {
622 GGML_LOG_ERROR("%s: failed to seek to beginning of data section\n", __func__);
623 gguf_free(ctx);
624 return nullptr;
625 }
626
627 // store the current file offset - this is where the data section starts
628 ctx->offset = ftell(file);
629
630 // compute the total size of the data section, taking into account the alignment
631 {
632 ctx->size = 0;
633 for (size_t i = 0; i < ctx->info.size(); ++i) {
634 const gguf_tensor_info & ti = ctx->info[i];
635 if (ti.offset != ctx->size) {
636 GGML_LOG_ERROR("%s: tensor '%s' has offset %" PRIu64 ", expected %zu\n",
637 __func__, ti.t.name, ti.offset, ctx->size);
638 GGML_LOG_ERROR("%s: failed to read tensor data\n", __func__);
639 gguf_free(ctx);
640 return nullptr;
641 }
642 size_t padded_size = GGML_PAD(ggml_nbytes(&ti.t), ctx->alignment);
643 if (SIZE_MAX - ctx->size < padded_size) {
644 GGML_LOG_ERROR("%s: tensor '%s' size overflow, cannot accumulate size %zu + %zu\n",
645 __func__, ti.t.name, ctx->size, padded_size);
646 gguf_free(ctx);
647 return nullptr;
648 }
649 ctx->size += padded_size;
650 }
651 }
652
653 // load the tensor data only if requested
654 if (params.ctx != nullptr) {
655 // if the provided gguf_context is no_alloc, then we create "empty" tensors and do not read the binary blob
656 // otherwise, we load the binary blob into the created ggml_context as well, and point the "data" members of
657 // the ggml_tensor structs to the appropriate locations in the binary blob
658
659 // compute the exact size needed for the new ggml_context
660 const size_t mem_size =
661 params.no_alloc ?
662 (n_tensors )*ggml_tensor_overhead() :
663 (n_tensors + 1)*ggml_tensor_overhead() + ctx->size;
664
665 struct ggml_init_params pdata = {
666 /*mem_size =*/ mem_size,
667 /*mem_buffer =*/ nullptr,
668 /*no_alloc =*/ params.no_alloc,
669 };
670
671 *params.ctx = ggml_init(pdata);
672 if (*params.ctx == nullptr) {
673 GGML_LOG_ERROR("%s: failed to initialize ggml context for storing tensors\n", __func__);
674 gguf_free(ctx);
675 return nullptr;
676 }
677
678 struct ggml_context * ctx_data = *params.ctx;
679
680 struct ggml_tensor * data = nullptr;
681
682 if (!params.no_alloc) {
683 data = ggml_new_tensor_1d(ctx_data, GGML_TYPE_I8, ctx->size);
684
685 ok = ok && data != nullptr;
686
687 if (ok) {
688 ggml_set_name(data, "GGUF tensor data binary blob");
689 }
690
691 // read the binary blob with the tensor data
692 ok = ok && gr.read(data->data, ctx->size);
693
694 if (!ok) {
695 GGML_LOG_ERROR("%s: failed to read tensor data binary blob\n", __func__);
696 ggml_free(ctx_data);
697 *params.ctx = nullptr;
698 gguf_free(ctx);
699 return nullptr;
700 }
701
702 ctx->data = data->data;
703 }
704
705 ggml_set_no_alloc(ctx_data, true);
706
707 // create the tensors
708 for (size_t i = 0; i < ctx->info.size(); ++i) {
709 const struct gguf_tensor_info & info = ctx->info[i];
710
711 struct ggml_tensor * cur = ggml_new_tensor(ctx_data, info.t.type, GGML_MAX_DIMS, info.t.ne);
712
713 ok = ok && cur != nullptr;
714
715 if (!ok) {
716 break;
717 }
718
719 ggml_set_name(cur, info.t.name);
720
721 // point the data member to the appropriate location in the binary blob using the tensor info
722 if (!params.no_alloc) {
723 cur->data = (char *) data->data + info.offset;
724 }
725 }
726
727 if (!ok) {
728 GGML_LOG_ERROR("%s: failed to create tensors\n", __func__);
729 ggml_free(ctx_data);
730 *params.ctx = nullptr;
731 gguf_free(ctx);
732 return nullptr;
733 }
734
735 ggml_set_no_alloc(ctx_data, params.no_alloc);
736 }
737
738 return ctx;
739}
740
741struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params) {
742 FILE * file = ggml_fopen(fname, "rb");
743
744 if (!file) {
745 GGML_LOG_ERROR("%s: failed to open GGUF file '%s' (%s)\n", __func__, fname, strerror(errno));
746 return nullptr;
747 }
748
749 struct gguf_context * result = gguf_init_from_file_impl(file, params);
750 fclose(file);
751 return result;
752}
753
754void gguf_free(struct gguf_context * ctx) {
755 if (ctx == nullptr) {
756 return;
757 }
758 delete ctx;
759}
760
761const char * gguf_type_name(enum gguf_type type) {
762 auto it = GGUF_TYPE_NAME.find(type);
763 return it == GGUF_TYPE_NAME.end() ? nullptr : it->second;
764}
765
766uint32_t gguf_get_version(const struct gguf_context * ctx) {
767 return ctx->version;
768}
769
770size_t gguf_get_alignment(const struct gguf_context * ctx) {
771 return ctx->alignment;
772}
773
774size_t gguf_get_data_offset(const struct gguf_context * ctx) {
775 return ctx->offset;
776}
777
778int64_t gguf_get_n_kv(const struct gguf_context * ctx) {
779 return ctx->kv.size();
780}
781
782int64_t gguf_find_key(const struct gguf_context * ctx, const char * key) {
783 // return -1 if key not found
784 int64_t keyfound = -1;
785
786 const int64_t n_kv = gguf_get_n_kv(ctx);
787
788 for (int64_t i = 0; i < n_kv; ++i) {
789 if (strcmp(key, gguf_get_key(ctx, i)) == 0) {
790 keyfound = i;
791 break;
792 }
793 }
794
795 return keyfound;
796}
797
798const char * gguf_get_key(const struct gguf_context * ctx, int64_t key_id) {
799 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
800 return ctx->kv[key_id].get_key().c_str();
801}
802
803enum gguf_type gguf_get_kv_type(const struct gguf_context * ctx, int64_t key_id) {
804 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
805 return ctx->kv[key_id].is_array ? GGUF_TYPE_ARRAY : ctx->kv[key_id].get_type();
806}
807
808enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int64_t key_id) {
809 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
810 GGML_ASSERT(ctx->kv[key_id].is_array);
811 return ctx->kv[key_id].get_type();
812}
813
814const void * gguf_get_arr_data(const struct gguf_context * ctx, int64_t key_id) {
815 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
816 GGML_ASSERT(ctx->kv[key_id].get_type() != GGUF_TYPE_STRING);
817 return ctx->kv[key_id].data.data();
818}
819
820const char * gguf_get_arr_str(const struct gguf_context * ctx, int64_t key_id, size_t i) {
821 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
822 GGML_ASSERT(ctx->kv[key_id].get_type() == GGUF_TYPE_STRING);
823 return ctx->kv[key_id].data_string[i].c_str();
824}
825
826size_t gguf_get_arr_n(const struct gguf_context * ctx, int64_t key_id) {
827 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
828
829 if (ctx->kv[key_id].type == GGUF_TYPE_STRING) {
830 return ctx->kv[key_id].data_string.size();
831 }
832
833 const size_t type_size = gguf_type_size(ctx->kv[key_id].type);
834 GGML_ASSERT(ctx->kv[key_id].data.size() % type_size == 0);
835 return ctx->kv[key_id].data.size() / type_size;
836}
837
838uint8_t gguf_get_val_u8(const struct gguf_context * ctx, int64_t key_id) {
839 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
840 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
841 return ctx->kv[key_id].get_val<uint8_t>();
842}
843
844int8_t gguf_get_val_i8(const struct gguf_context * ctx, int64_t key_id) {
845 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
846 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
847 return ctx->kv[key_id].get_val<int8_t>();
848}
849
850uint16_t gguf_get_val_u16(const struct gguf_context * ctx, int64_t key_id) {
851 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
852 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
853 return ctx->kv[key_id].get_val<uint16_t>();
854}
855
856int16_t gguf_get_val_i16(const struct gguf_context * ctx, int64_t key_id) {
857 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
858 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
859 return ctx->kv[key_id].get_val<int16_t>();
860}
861
862uint32_t gguf_get_val_u32(const struct gguf_context * ctx, int64_t key_id) {
863 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
864 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
865 return ctx->kv[key_id].get_val<uint32_t>();
866}
867
868int32_t gguf_get_val_i32(const struct gguf_context * ctx, int64_t key_id) {
869 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
870 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
871 return ctx->kv[key_id].get_val<int32_t>();
872}
873
874float gguf_get_val_f32(const struct gguf_context * ctx, int64_t key_id) {
875 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
876 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
877 return ctx->kv[key_id].get_val<float>();
878}
879
880uint64_t gguf_get_val_u64(const struct gguf_context * ctx, int64_t key_id) {
881 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
882 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
883 return ctx->kv[key_id].get_val<uint64_t>();
884}
885
886int64_t gguf_get_val_i64(const struct gguf_context * ctx, int64_t key_id) {
887 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
888 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
889 return ctx->kv[key_id].get_val<int64_t>();
890}
891
892double gguf_get_val_f64(const struct gguf_context * ctx, int64_t key_id) {
893 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
894 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
895 return ctx->kv[key_id].get_val<double>();
896}
897
898bool gguf_get_val_bool(const struct gguf_context * ctx, int64_t key_id) {
899 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
900 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
901 return ctx->kv[key_id].get_val<bool>();
902}
903
904const char * gguf_get_val_str(const struct gguf_context * ctx, int64_t key_id) {
905 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
906 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
907 return ctx->kv[key_id].get_val<std::string>().c_str();
908}
909
910const void * gguf_get_val_data(const struct gguf_context * ctx, int64_t key_id) {
911 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));
912 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);
913 GGML_ASSERT(ctx->kv[key_id].get_type() != GGUF_TYPE_STRING);
914 return ctx->kv[key_id].data.data();
915}
916
917int64_t gguf_get_n_tensors(const struct gguf_context * ctx) {
918 return ctx->info.size();
919}
920
921int64_t gguf_find_tensor(const struct gguf_context * ctx, const char * name) {
922 // return -1 if tensor not found
923 int64_t tensor_id = -1;
924
925 const int64_t n_tensors = gguf_get_n_tensors(ctx);
926
927 for (int64_t i = 0; i < n_tensors; ++i) {
928 if (strcmp(name, gguf_get_tensor_name(ctx, i)) == 0) {
929 tensor_id = i;
930 break;
931 }
932 }
933
934 return tensor_id;
935}
936
937size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int64_t tensor_id) {
938 GGML_ASSERT(tensor_id >= 0 && tensor_id < gguf_get_n_tensors(ctx));
939 return ctx->info[tensor_id].offset;
940}
941
942const char * gguf_get_tensor_name(const struct gguf_context * ctx, int64_t tensor_id) {
943 GGML_ASSERT(tensor_id >= 0 && tensor_id < gguf_get_n_tensors(ctx));
944 return ctx->info[tensor_id].t.name;
945}
946
947enum ggml_type gguf_get_tensor_type(const struct gguf_context * ctx, int64_t tensor_id) {
948 GGML_ASSERT(tensor_id >= 0 && tensor_id < gguf_get_n_tensors(ctx));
949 return ctx->info[tensor_id].t.type;
950}
951
952size_t gguf_get_tensor_size(const struct gguf_context * ctx, int64_t tensor_id) {
953 GGML_ASSERT(tensor_id >= 0 && tensor_id < gguf_get_n_tensors(ctx));
954 return ggml_nbytes(&ctx->info[tensor_id].t);
955}
956
957int64_t gguf_remove_key(struct gguf_context * ctx, const char * key) {
958 const int64_t key_id = gguf_find_key(ctx, key);
959 if (key_id >= 0) {
960 ctx->kv.erase(ctx->kv.begin() + key_id);
961 }
962 return key_id;
963}
964
965template<typename T>
966static void gguf_check_reserved_keys(const std::string & key, const T val) {
967 if (key == GGUF_KEY_GENERAL_ALIGNMENT) {
968 if constexpr (std::is_same<T, uint32_t>::value) {
969 GGML_ASSERT(val > 0 && (val & (val - 1)) == 0 && GGUF_KEY_GENERAL_ALIGNMENT " must be power of 2");
970 } else {
971 GGML_UNUSED(val);
972 GGML_ABORT(GGUF_KEY_GENERAL_ALIGNMENT " must be type u32");
973 }
974 }
975}
976
977void gguf_set_val_u8(struct gguf_context * ctx, const char * key, uint8_t val) {
978 gguf_check_reserved_keys(key, val);
979 gguf_remove_key(ctx, key);
980 ctx->kv.emplace_back(key, val);
981}
982
983void gguf_set_val_i8(struct gguf_context * ctx, const char * key, int8_t val) {
984 gguf_check_reserved_keys(key, val);
985 gguf_remove_key(ctx, key);
986 ctx->kv.emplace_back(key, val);
987}
988
989void gguf_set_val_u16(struct gguf_context * ctx, const char * key, uint16_t val) {
990 gguf_check_reserved_keys(key, val);
991 gguf_remove_key(ctx, key);
992 ctx->kv.emplace_back(key, val);
993}
994
995void gguf_set_val_i16(struct gguf_context * ctx, const char * key, int16_t val) {
996 gguf_check_reserved_keys(key, val);
997 gguf_remove_key(ctx, key);
998 ctx->kv.emplace_back(key, val);
999}
1000
1001void gguf_set_val_u32(struct gguf_context * ctx, const char * key, uint32_t val) {
1002 gguf_check_reserved_keys(key, val);
1003 gguf_remove_key(ctx, key);
1004 ctx->kv.emplace_back(key, val);
1005}
1006
1007void gguf_set_val_i32(struct gguf_context * ctx, const char * key, int32_t val) {
1008 gguf_check_reserved_keys(key, val);
1009 gguf_remove_key(ctx, key);
1010 ctx->kv.emplace_back(key, val);
1011}
1012
1013void gguf_set_val_f32(struct gguf_context * ctx, const char * key, float val) {
1014 gguf_check_reserved_keys(key, val);
1015 gguf_remove_key(ctx, key);
1016 ctx->kv.emplace_back(key, val);
1017}
1018
1019void gguf_set_val_u64(struct gguf_context * ctx, const char * key, uint64_t val) {
1020 gguf_check_reserved_keys(key, val);
1021 gguf_remove_key(ctx, key);
1022 ctx->kv.emplace_back(key, val);
1023}
1024
1025void gguf_set_val_i64(struct gguf_context * ctx, const char * key, int64_t val) {
1026 gguf_check_reserved_keys(key, val);
1027 gguf_remove_key(ctx, key);
1028 ctx->kv.emplace_back(key, val);
1029}
1030
1031void gguf_set_val_f64(struct gguf_context * ctx, const char * key, double val) {
1032 gguf_check_reserved_keys(key, val);
1033 gguf_remove_key(ctx, key);
1034 ctx->kv.emplace_back(key, val);
1035}
1036
1037void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool val) {
1038 gguf_check_reserved_keys(key, val);
1039 gguf_remove_key(ctx, key);
1040 ctx->kv.emplace_back(key, val);
1041}
1042
1043void gguf_set_val_str(struct gguf_context * ctx, const char * key, const char * val) {
1044 gguf_check_reserved_keys(key, val);
1045 gguf_remove_key(ctx, key);
1046 ctx->kv.emplace_back(key, std::string(val));
1047}
1048
1049void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, size_t n) {
1050 gguf_check_reserved_keys(key, data);
1051 gguf_remove_key(ctx, key);
1052
1053 const size_t nbytes = n*gguf_type_size(type);
1054 std::vector<int8_t> tmp(nbytes);
1055 if (!tmp.empty()) {
1056 memcpy(tmp.data(), data, nbytes);
1057 }
1058 ctx->kv.emplace_back(key, tmp);
1059 ctx->kv.back().cast(type);
1060}
1061
1062void gguf_set_arr_str(struct gguf_context * ctx, const char * key, const char ** data, size_t n) {
1063 gguf_check_reserved_keys(key, data);
1064 gguf_remove_key(ctx, key);
1065
1066 std::vector<std::string> tmp(n);
1067 for (size_t i = 0; i < n; ++i) {
1068 tmp[i] = data[i];
1069 }
1070 ctx->kv.emplace_back(key, tmp);
1071}
1072
1073// set or add KV pairs from another context
1074void gguf_set_kv(struct gguf_context * ctx, const struct gguf_context * src) {
1075 const int64_t n_kv = gguf_get_n_kv(src);
1076 for (int64_t i = 0; i < n_kv; ++i) {
1077 const struct gguf_kv & kv = src->kv[i];
1078
1079 if (!kv.is_array) {
1080 switch (kv.get_type()) {
1081 case GGUF_TYPE_UINT8: gguf_set_val_u8 (ctx, kv.get_key().c_str(), kv.get_val<uint8_t>()); break;
1082 case GGUF_TYPE_INT8: gguf_set_val_i8 (ctx, kv.get_key().c_str(), kv.get_val<int8_t>()); break;
1083 case GGUF_TYPE_UINT16: gguf_set_val_u16 (ctx, kv.get_key().c_str(), kv.get_val<uint16_t>()); break;
1084 case GGUF_TYPE_INT16: gguf_set_val_i16 (ctx, kv.get_key().c_str(), kv.get_val<int16_t>()); break;
1085 case GGUF_TYPE_UINT32: gguf_set_val_u32 (ctx, kv.get_key().c_str(), kv.get_val<uint32_t>()); break;
1086 case GGUF_TYPE_INT32: gguf_set_val_i32 (ctx, kv.get_key().c_str(), kv.get_val<int32_t>()); break;
1087 case GGUF_TYPE_FLOAT32: gguf_set_val_f32 (ctx, kv.get_key().c_str(), kv.get_val<float>()); break;
1088 case GGUF_TYPE_UINT64: gguf_set_val_u64 (ctx, kv.get_key().c_str(), kv.get_val<uint64_t>()); break;
1089 case GGUF_TYPE_INT64: gguf_set_val_i64 (ctx, kv.get_key().c_str(), kv.get_val<int64_t>()); break;
1090 case GGUF_TYPE_FLOAT64: gguf_set_val_f64 (ctx, kv.get_key().c_str(), kv.get_val<double>()); break;
1091 case GGUF_TYPE_BOOL: gguf_set_val_bool(ctx, kv.get_key().c_str(), kv.get_val<bool>()); break;
1092 case GGUF_TYPE_STRING: gguf_set_val_str (ctx, kv.get_key().c_str(), kv.get_val<std::string>().c_str()); break;
1093 case GGUF_TYPE_ARRAY:
1094 default: GGML_ABORT("invalid type");
1095 }
1096 continue;
1097 }
1098
1099 const size_t ne = kv.get_ne();
1100
1101 switch (kv.get_type()) {
1102 case GGUF_TYPE_UINT8:
1103 case GGUF_TYPE_INT8:
1104 case GGUF_TYPE_UINT16:
1105 case GGUF_TYPE_INT16:
1106 case GGUF_TYPE_UINT32:
1107 case GGUF_TYPE_INT32:
1108 case GGUF_TYPE_FLOAT32:
1109 case GGUF_TYPE_UINT64:
1110 case GGUF_TYPE_INT64:
1111 case GGUF_TYPE_FLOAT64:
1112 case GGUF_TYPE_BOOL: {
1113 gguf_set_arr_data(ctx, kv.get_key().c_str(), kv.get_type(), kv.data.data(), ne);
1114 } break;
1115 case GGUF_TYPE_STRING: {
1116 std::vector<const char *> tmp(ne);
1117 for (size_t j = 0; j < ne; ++j) {
1118 tmp[j] = kv.data_string[j].c_str();
1119 }
1120 gguf_set_arr_str(ctx, kv.get_key().c_str(), tmp.data(), ne);
1121 } break;
1122 case GGUF_TYPE_ARRAY:
1123 default: GGML_ABORT("invalid type");
1124 }
1125 }
1126}
1127
1128void gguf_add_tensor(
1129 struct gguf_context * ctx,
1130 const struct ggml_tensor * tensor) {
1131 GGML_ASSERT(tensor);
1132 if (gguf_find_tensor(ctx, tensor->name) != -1) {
1133 GGML_ABORT("duplicate tensor name: %s", tensor->name);
1134 }
1135
1136 struct gguf_tensor_info ti;
1137 ti.t = *tensor;
1138 ti.offset = ctx->info.empty() ? 0 :
1139 ctx->info.back().offset + GGML_PAD(ggml_nbytes(&ctx->info.back().t), ctx->alignment);
1140 ctx->info.push_back(ti);
1141}
1142
1143void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type) {
1144 const int64_t tensor_id = gguf_find_tensor(ctx, name);
1145 if (tensor_id < 0) {
1146 GGML_ABORT("tensor not found: %s", name);
1147 }
1148 struct ggml_tensor * tensor = &ctx->info[tensor_id].t;
1149 const size_t type_size = ggml_type_size(type);
1150 const int64_t blck_size = ggml_blck_size(type);
1151
1152 tensor->type = type;
1153 GGML_ASSERT(tensor->ne[0] % blck_size == 0 && "tensor row size not divisible by block size of new type");
1154
1155 tensor->nb[0] = type_size;
1156 tensor->nb[1] = tensor->nb[0]*(tensor->ne[0]/blck_size);
1157 for (int i = 2; i < GGML_MAX_DIMS; i++) {
1158 tensor->nb[i] = tensor->nb[i - 1]*tensor->ne[i - 1];
1159 }
1160
1161 // update offsets
1162 const int64_t n_tensors = gguf_get_n_tensors(ctx);
1163 for (int64_t i = tensor_id + 1; i < n_tensors; ++i) {
1164 ctx->info[i].offset = ctx->info[i - 1].offset + GGML_PAD(ggml_nbytes(&ctx->info[i - 1].t), ctx->alignment);
1165 }
1166}
1167
1168void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data) {
1169 const int64_t tensor_id = gguf_find_tensor(ctx, name);
1170 if (tensor_id < 0) {
1171 GGML_ABORT("tensor not found: %s", name);
1172 }
1173
1174 ctx->info[tensor_id].t.data = (void *)(uintptr_t)data; // double cast suppresses warning about casting away const
1175}
1176
1177struct gguf_writer_base {
1178 size_t written_bytes {0u};
1179
1180 ~gguf_writer_base(void) = default;
1181
1182 // we bet on devirtualization
1183 virtual void write(int8_t val) = 0;
1184 virtual void write(const std::vector<int8_t> & val) = 0;
1185 virtual void write_tensor_data(const struct gguf_tensor_info & info, size_t offset_data, size_t alignment) = 0;
1186
1187 template <typename T>
1188 void write(const T & val) {
1189 for (size_t i = 0; i < sizeof(val); ++i) {
1190 write(reinterpret_cast<const int8_t *>(&val)[i]);
1191 }
1192 }
1193
1194 void write(const bool & val) {
1195 const int8_t val8 = val ? 1 : 0;
1196 write(val8);
1197 }
1198
1199 void write(const std::string & val) {
1200 {
1201 const uint64_t n = val.length();
1202 write(n);
1203 }
1204 for (size_t i = 0; i < val.length(); ++i) {
1205 write((val.data())[i]);
1206 }
1207 }
1208
1209 void write(const char * val) {
1210 write(std::string(val));
1211 }
1212
1213 void write(const enum ggml_type & val) {
1214 write(int32_t(val));
1215 }
1216
1217 void write(const enum gguf_type & val) {
1218 write(int32_t(val));
1219 }
1220
1221 void write(const struct gguf_kv & kv) {
1222 const uint64_t ne = kv.get_ne();
1223
1224 write(kv.get_key());
1225
1226 if (kv.is_array) {
1227 write(GGUF_TYPE_ARRAY);
1228 write(kv.get_type());
1229 write(ne);
1230 } else {
1231 write(kv.get_type());
1232 }
1233
1234 switch (kv.get_type()) {
1235 case GGUF_TYPE_UINT8:
1236 case GGUF_TYPE_INT8:
1237 case GGUF_TYPE_UINT16:
1238 case GGUF_TYPE_INT16:
1239 case GGUF_TYPE_UINT32:
1240 case GGUF_TYPE_INT32:
1241 case GGUF_TYPE_FLOAT32:
1242 case GGUF_TYPE_UINT64:
1243 case GGUF_TYPE_INT64:
1244 case GGUF_TYPE_FLOAT64: {
1245 write(kv.data);
1246 } break;
1247 case GGUF_TYPE_BOOL: {
1248 for (size_t i = 0; i < ne; ++i) {
1249 write(kv.get_val<bool>(i));
1250 }
1251 } break;
1252 case GGUF_TYPE_STRING: {
1253 for (size_t i = 0; i < ne; ++i) {
1254 write(kv.get_val<std::string>(i));
1255 }
1256 } break;
1257 case GGUF_TYPE_ARRAY:
1258 default: GGML_ABORT("invalid type");
1259 }
1260 }
1261
1262 void write_tensor_meta(const struct gguf_tensor_info & info) {
1263 write(info.t.name);
1264
1265 const uint32_t n_dims = ggml_n_dims(&info.t);
1266 write(n_dims);
1267
1268 for (uint32_t j = 0; j < n_dims; ++j) {
1269 write(info.t.ne[j]);
1270 }
1271 write(info.t.type);
1272 write(info.offset);
1273 }
1274
1275 void pad(const size_t alignment) {
1276 while (written_bytes % alignment != 0) {
1277 const int8_t zero = 0;
1278 write(zero);
1279 }
1280 }
1281};
1282
1283// vector buffer based writer
1284struct gguf_writer_buf final : public gguf_writer_base {
1285 std::vector<int8_t> & buf;
1286
1287 gguf_writer_buf(std::vector<int8_t> & buf) : buf(buf) {}
1288
1289 using gguf_writer_base::write;
1290
1291 void write(const int8_t val) override {
1292 buf.push_back(val);
1293 written_bytes++;
1294 }
1295
1296 void write(const std::vector<int8_t> & val) override {
1297 buf.insert(buf.end(), val.begin(), val.end());
1298 written_bytes += val.size();
1299 }
1300
1301 void write_tensor_data(const struct gguf_tensor_info & info, const size_t offset_data, const size_t alignment) override {
1302 GGML_ASSERT(buf.size() - offset_data == info.offset);
1303
1304 GGML_ASSERT(ggml_is_contiguous(&info.t));
1305 const size_t offset = buf.size();
1306 const size_t nbytes = ggml_nbytes(&info.t);
1307
1308 buf.resize(offset + nbytes);
1309 if (info.t.buffer) {
1310 ggml_backend_tensor_get(&info.t, buf.data() + offset, 0, nbytes);
1311 } else {
1312 GGML_ASSERT(info.t.data);
1313 memcpy(buf.data() + offset, info.t.data, nbytes);
1314 }
1315 written_bytes += nbytes;
1316
1317 pad(alignment);
1318 }
1319};
1320
1321// file based writer
1322struct gguf_writer_file final : public gguf_writer_base {
1323 FILE * file;
1324
1325 gguf_writer_file(FILE* file) : file(file) {}
1326
1327 using gguf_writer_base::write;
1328
1329 void write(const int8_t val) override {
1330 const auto real_val = static_cast<uint8_t>(val);
1331 const auto ret = fputc(real_val, file);
1332 written_bytes++;
1333 if (ret != real_val) {
1334 throw std::runtime_error("unexpected fputc result '" + std::to_string(ret) + "' instead of '" + std::to_string((int)real_val) + "'");
1335 }
1336 }
1337
1338 void write(const std::vector<int8_t> & val) override {
1339 const auto ret = fwrite(val.data(), 1, val.size(), file);
1340 written_bytes += val.size();
1341 if (ret != val.size()) {
1342 throw std::runtime_error("unexpected fwrite number of bytes written, '" + std::to_string(ret) + "' instead of '" + std::to_string(val.size()) + "'");
1343 }
1344 }
1345
1346 void write_tensor_data(const struct gguf_tensor_info & info, const size_t offset_data, const size_t alignment) override {
1347 GGML_ASSERT(written_bytes - offset_data == info.offset);
1348
1349 GGML_ASSERT(ggml_is_contiguous(&info.t));
1350 const size_t nbytes = ggml_nbytes(&info.t);
1351
1352 std::vector<int8_t> buf(nbytes);
1353 if (info.t.buffer) {
1354 ggml_backend_tensor_get(&info.t, buf.data(), 0, nbytes);
1355 } else {
1356 GGML_ASSERT(info.t.data);
1357 memcpy(buf.data(), info.t.data, nbytes);
1358 }
1359 write(buf);
1360
1361 pad(alignment);
1362 }
1363};
1364
1365template <typename writer_t>
1366static void gguf_write_out(const struct gguf_context * ctx, writer_t & gw, bool only_meta) {
1367 const int64_t n_kv = gguf_get_n_kv(ctx);
1368 const int64_t n_tensors = gguf_get_n_tensors(ctx);
1369
1370 // write header
1371 gw.write(GGUF_MAGIC[0]);
1372 gw.write(GGUF_MAGIC[1]);
1373 gw.write(GGUF_MAGIC[2]);
1374 gw.write(GGUF_MAGIC[3]);
1375 gw.write(ctx->version);
1376 gw.write(n_tensors);
1377 gw.write(n_kv);
1378
1379 // write key-value pairs
1380 for (int64_t i = 0; i < n_kv; ++i) {
1381 gw.write(ctx->kv[i]);
1382 }
1383
1384 // write tensor info
1385 for (int64_t i = 0; i < n_tensors; ++i) {
1386 gw.write_tensor_meta(ctx->info[i]);
1387 }
1388
1389 // we require the data section to be aligned
1390 gw.pad(ctx->alignment);
1391
1392 if (only_meta) {
1393 return;
1394 }
1395
1396 const size_t offset_data = gw.written_bytes;
1397
1398 // write tensor data
1399 for (int64_t i = 0; i < n_tensors; ++i) {
1400 gw.write_tensor_data(ctx->info[i], offset_data, ctx->alignment);
1401 }
1402}
1403
1404void gguf_write_to_buf(const struct gguf_context * ctx, std::vector<int8_t> & buf, bool only_meta) {
1405 gguf_writer_buf gw(buf);
1406 gguf_write_out(ctx, gw, only_meta);
1407}
1408
1409bool gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta) {
1410 FILE * file = ggml_fopen(fname, "wb");
1411
1412 if (!file) {
1413 GGML_LOG_ERROR("%s: failed to open file '%s' for writing GGUF data\n", __func__, fname);
1414 return false;
1415 }
1416
1417 try {
1418 gguf_writer_file gw(file);
1419 gguf_write_out(ctx, gw, only_meta);
1420 } catch (const std::runtime_error& ex) {
1421 GGML_LOG_ERROR("%s: failed to write GGUF data into '%s': %s\n", __func__, fname, ex.what());
1422 fclose(file);
1423 return false;
1424 }
1425
1426 fclose(file);
1427 return true;
1428}
1429
1430size_t gguf_get_meta_size(const struct gguf_context * ctx) {
1431 // only return size
1432 std::vector<int8_t> buf;
1433 gguf_write_to_buf(ctx, buf, /*only_meta =*/ true);
1434 return buf.size();
1435}
1436
1437void gguf_get_meta_data(const struct gguf_context * ctx, void * data) {
1438 std::vector<int8_t> buf;
1439 gguf_write_to_buf(ctx, buf, /*only_meta =*/ true);
1440 memcpy(data, buf.data(), buf.size());
1441}