1#pragma once
2
3#include "ggml.h"
4#include "ggml-alloc.h"
5
6#ifdef GGML_BACKEND_SHARED
7# if defined(_WIN32) && !defined(__MINGW32__)
8# ifdef GGML_BACKEND_BUILD
9# define GGML_BACKEND_API __declspec(dllexport) extern
10# else
11# define GGML_BACKEND_API __declspec(dllimport) extern
12# endif
13# else
14# define GGML_BACKEND_API __attribute__ ((visibility ("default"))) extern
15# endif
16#else
17# define GGML_BACKEND_API extern
18#endif
19
20#ifdef __cplusplus
21extern "C" {
22#endif
23
24 typedef struct ggml_backend_buffer_type * ggml_backend_buffer_type_t;
25 typedef struct ggml_backend_buffer * ggml_backend_buffer_t;
26 typedef struct ggml_backend_event * ggml_backend_event_t;
27 typedef struct ggml_backend * ggml_backend_t;
28 typedef void * ggml_backend_graph_plan_t;
29 typedef struct ggml_backend_reg * ggml_backend_reg_t;
30 typedef struct ggml_backend_device * ggml_backend_dev_t;
31
32
33 //
34 // Backend buffer type
35 //
36
37 GGML_API const char * ggml_backend_buft_name (ggml_backend_buffer_type_t buft);
38 GGML_API ggml_backend_buffer_t ggml_backend_buft_alloc_buffer (ggml_backend_buffer_type_t buft, size_t size);
39 GGML_API size_t ggml_backend_buft_get_alignment (ggml_backend_buffer_type_t buft);
40 GGML_API size_t ggml_backend_buft_get_max_size (ggml_backend_buffer_type_t buft);
41 GGML_API size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const struct ggml_tensor * tensor);
42 GGML_API bool ggml_backend_buft_is_host (ggml_backend_buffer_type_t buft);
43 GGML_API ggml_backend_dev_t ggml_backend_buft_get_device (ggml_backend_buffer_type_t buft);
44
45 //
46 // Backend buffer
47 //
48
49 enum ggml_backend_buffer_usage {
50 GGML_BACKEND_BUFFER_USAGE_ANY = 0,
51 GGML_BACKEND_BUFFER_USAGE_WEIGHTS = 1,
52 GGML_BACKEND_BUFFER_USAGE_COMPUTE = 2,
53 };
54
55 GGML_API const char * ggml_backend_buffer_name (ggml_backend_buffer_t buffer);
56 GGML_API void ggml_backend_buffer_free (ggml_backend_buffer_t buffer);
57 GGML_API void * ggml_backend_buffer_get_base (ggml_backend_buffer_t buffer);
58 GGML_API size_t ggml_backend_buffer_get_size (ggml_backend_buffer_t buffer);
59 GGML_API enum ggml_status ggml_backend_buffer_init_tensor (ggml_backend_buffer_t buffer, struct ggml_tensor * tensor);
60 GGML_API size_t ggml_backend_buffer_get_alignment (ggml_backend_buffer_t buffer);
61 GGML_API size_t ggml_backend_buffer_get_max_size (ggml_backend_buffer_t buffer);
62 GGML_API size_t ggml_backend_buffer_get_alloc_size(ggml_backend_buffer_t buffer, const struct ggml_tensor * tensor);
63 GGML_API void ggml_backend_buffer_clear (ggml_backend_buffer_t buffer, uint8_t value);
64 GGML_API bool ggml_backend_buffer_is_host (ggml_backend_buffer_t buffer);
65 GGML_API void ggml_backend_buffer_set_usage (ggml_backend_buffer_t buffer, enum ggml_backend_buffer_usage usage);
66 GGML_API enum ggml_backend_buffer_usage ggml_backend_buffer_get_usage (ggml_backend_buffer_t buffer);
67 GGML_API ggml_backend_buffer_type_t ggml_backend_buffer_get_type (ggml_backend_buffer_t buffer);
68 GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer);
69
70 // tensor copy between different backends
71 GGML_API void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst);
72
73 //
74 // Backend (stream)
75 //
76
77 GGML_API ggml_guid_t ggml_backend_guid(ggml_backend_t backend);
78 GGML_API const char * ggml_backend_name(ggml_backend_t backend);
79 GGML_API void ggml_backend_free(ggml_backend_t backend);
80
81 GGML_API ggml_backend_buffer_type_t ggml_backend_get_default_buffer_type(ggml_backend_t backend);
82 GGML_API ggml_backend_buffer_t ggml_backend_alloc_buffer(ggml_backend_t backend, size_t size);
83 GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend);
84 GGML_API size_t ggml_backend_get_max_size(ggml_backend_t backend);
85
86 GGML_API void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
87 GGML_API void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
88
89 // "offset" refers to the offset in tensor->data for setting/getting data
90 GGML_API void ggml_backend_tensor_set( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size);
91 GGML_API void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size);
92 GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size);
93
94 GGML_API void ggml_backend_synchronize(ggml_backend_t backend);
95
96 GGML_API ggml_backend_graph_plan_t ggml_backend_graph_plan_create(ggml_backend_t backend, struct ggml_cgraph * cgraph);
97 GGML_API void ggml_backend_graph_plan_free (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
98
99 GGML_API enum ggml_status ggml_backend_graph_plan_compute (ggml_backend_t backend, ggml_backend_graph_plan_t plan);
100 GGML_API enum ggml_status ggml_backend_graph_compute (ggml_backend_t backend, struct ggml_cgraph * cgraph);
101 GGML_API enum ggml_status ggml_backend_graph_compute_async(ggml_backend_t backend, struct ggml_cgraph * cgraph);
102
103 // NOTE: will be removed, use device version instead
104 GGML_API bool ggml_backend_supports_op(ggml_backend_t backend, const struct ggml_tensor * op);
105 GGML_API bool ggml_backend_supports_buft(ggml_backend_t backend, ggml_backend_buffer_type_t buft);
106 GGML_API bool ggml_backend_offload_op(ggml_backend_t backend, const struct ggml_tensor * op);
107
108 // asynchronous copy
109 // the copy is performed after all the currently queued operations in backend_src
110 // backend_dst will wait for the copy to complete before performing other operations
111 // automatic fallback to sync copy if async is not supported
112 GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst);
113
114 GGML_API ggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend);
115
116 //
117 // Events
118 //
119
120 GGML_API ggml_backend_event_t ggml_backend_event_new(ggml_backend_dev_t device);
121 GGML_API void ggml_backend_event_free(ggml_backend_event_t event);
122 GGML_API void ggml_backend_event_record(ggml_backend_event_t event, ggml_backend_t backend);
123 GGML_API void ggml_backend_event_synchronize(ggml_backend_event_t event);
124 GGML_API void ggml_backend_event_wait(ggml_backend_t backend, ggml_backend_event_t event);
125
126 //
127 // Backend device
128 //
129
130 enum ggml_backend_dev_type {
131 // CPU device using system memory
132 GGML_BACKEND_DEVICE_TYPE_CPU,
133 // GPU device using dedicated memory
134 GGML_BACKEND_DEVICE_TYPE_GPU,
135 // integrated GPU device using host memory
136 GGML_BACKEND_DEVICE_TYPE_IGPU,
137 // accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX)
138 GGML_BACKEND_DEVICE_TYPE_ACCEL
139 };
140
141 // functionality supported by the device
142 struct ggml_backend_dev_caps {
143 // asynchronous operations
144 bool async;
145 // pinned host buffer
146 bool host_buffer;
147 // creating buffers from host ptr
148 bool buffer_from_host_ptr;
149 // event synchronization
150 bool events;
151 };
152
153 // all the device properties
154 struct ggml_backend_dev_props {
155 // device name
156 const char * name;
157 // device description
158 const char * description;
159 // device free memory in bytes
160 size_t memory_free;
161 // device total memory in bytes
162 size_t memory_total;
163 // device type
164 enum ggml_backend_dev_type type;
165 // device id
166 // for PCI devices, this should be the PCI bus id formatted as "domain:bus:device.function" (e.g. "0000:01:00.0")
167 // if the id is unknown, this should be NULL
168 const char * device_id;
169 // device capabilities
170 struct ggml_backend_dev_caps caps;
171 };
172
173 GGML_API const char * ggml_backend_dev_name(ggml_backend_dev_t device);
174 GGML_API const char * ggml_backend_dev_description(ggml_backend_dev_t device);
175 GGML_API void ggml_backend_dev_memory(ggml_backend_dev_t device, size_t * free, size_t * total);
176 GGML_API enum ggml_backend_dev_type ggml_backend_dev_type(ggml_backend_dev_t device);
177 GGML_API void ggml_backend_dev_get_props(ggml_backend_dev_t device, struct ggml_backend_dev_props * props);
178 GGML_API ggml_backend_reg_t ggml_backend_dev_backend_reg(ggml_backend_dev_t device);
179 GGML_API ggml_backend_t ggml_backend_dev_init(ggml_backend_dev_t device, const char * params);
180 GGML_API ggml_backend_buffer_type_t ggml_backend_dev_buffer_type(ggml_backend_dev_t device);
181 GGML_API ggml_backend_buffer_type_t ggml_backend_dev_host_buffer_type(ggml_backend_dev_t device);
182 GGML_API ggml_backend_buffer_t ggml_backend_dev_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size);
183
184 GGML_API bool ggml_backend_dev_supports_op(ggml_backend_dev_t device, const struct ggml_tensor * op);
185 GGML_API bool ggml_backend_dev_supports_buft(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft);
186 GGML_API bool ggml_backend_dev_offload_op(ggml_backend_dev_t device, const struct ggml_tensor * op);
187
188 //
189 // Backend (reg)
190 //
191
192 GGML_API const char * ggml_backend_reg_name(ggml_backend_reg_t reg);
193 GGML_API size_t ggml_backend_reg_dev_count(ggml_backend_reg_t reg);
194 GGML_API ggml_backend_dev_t ggml_backend_reg_dev_get(ggml_backend_reg_t reg, size_t index);
195 GGML_API void * ggml_backend_reg_get_proc_address(ggml_backend_reg_t reg, const char * name);
196
197 // Common functions that may be obtained using ggml_backend_reg_get_proc_address
198
199 // Split buffer type for tensor parallelism
200 typedef ggml_backend_buffer_type_t (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split);
201 // Set the number of threads for the backend
202 typedef void (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads);
203 // Get additional buffer types provided by the device (returns a NULL-terminated array)
204 typedef ggml_backend_buffer_type_t * (*ggml_backend_dev_get_extra_bufts_t)(ggml_backend_dev_t device);
205 // Set the abort callback for the backend
206 typedef void (*ggml_backend_set_abort_callback_t)(ggml_backend_t backend, ggml_abort_callback abort_callback, void * abort_callback_data);
207 // Get a list of feature flags supported by the backend (returns a NULL-terminated array)
208 struct ggml_backend_feature {
209 const char * name;
210 const char * value;
211 };
212 typedef struct ggml_backend_feature * (*ggml_backend_get_features_t)(ggml_backend_reg_t reg);
213
214 //
215 // Backend registry
216 //
217
218 GGML_API void ggml_backend_register(ggml_backend_reg_t reg);
219
220 GGML_API void ggml_backend_device_register(ggml_backend_dev_t device);
221
222 // Backend (reg) enumeration
223 GGML_API size_t ggml_backend_reg_count(void);
224 GGML_API ggml_backend_reg_t ggml_backend_reg_get(size_t index);
225 GGML_API ggml_backend_reg_t ggml_backend_reg_by_name(const char * name);
226
227 // Device enumeration
228 GGML_API size_t ggml_backend_dev_count(void);
229 GGML_API ggml_backend_dev_t ggml_backend_dev_get(size_t index);
230 GGML_API ggml_backend_dev_t ggml_backend_dev_by_name(const char * name);
231 GGML_API ggml_backend_dev_t ggml_backend_dev_by_type(enum ggml_backend_dev_type type);
232
233 // Direct backend (stream) initialization
234 // = ggml_backend_dev_init(ggml_backend_dev_by_name(name), params)
235 GGML_API ggml_backend_t ggml_backend_init_by_name(const char * name, const char * params);
236 // = ggml_backend_dev_init(ggml_backend_dev_by_type(type), params)
237 GGML_API ggml_backend_t ggml_backend_init_by_type(enum ggml_backend_dev_type type, const char * params);
238 // = ggml_backend_dev_init(ggml_backend_dev_by_type(GPU) OR ggml_backend_dev_by_type(CPU), NULL)
239 GGML_API ggml_backend_t ggml_backend_init_best(void);
240
241 // Load a backend from a dynamic library and register it
242 GGML_API ggml_backend_reg_t ggml_backend_load(const char * path);
243 // Unload a backend if loaded dynamically and unregister it
244 GGML_API void ggml_backend_unload(ggml_backend_reg_t reg);
245 // Load all known backends from dynamic libraries
246 GGML_API void ggml_backend_load_all(void);
247 GGML_API void ggml_backend_load_all_from_path(const char * dir_path);
248
249 //
250 // Backend scheduler
251 //
252
253 // The backend scheduler allows for multiple backend devices to be used together
254 // Handles compute buffer allocation, assignment of tensors to backends, and copying of tensors between backends
255 // The backends are selected based on:
256 // - the backend that supports the operation
257 // - the location of the pre-allocated tensors (e.g. the weights)
258 /*
259 Example usage:
260
261 // operations that use tensors allocated in a buffer with USAGE_WEIGHTS will be assigned
262 // preferrably to run on the same backend as the buffer
263 ggml_backend_buffer_set_usage(buf_weights, GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
264
265 sched = ggml_backend_sched_new({backend_gpu, backend_gpu2, backend_cpu}, NULL, num_backends, GGML_DEFAULT_GRAPH_SIZE, false, true);
266
267 // initialize buffers from a max size graph (optional)
268 reserve_graph = build_graph(sched, max_batch_size);
269
270 // manually assign nodes to a backend (optional, should not be needed in most cases)
271 struct ggml_tensor * node = ggml_mul_mat(ctx, ...);
272 ggml_backend_sched_set_tensor_backend(sched, node, backend_gpu);
273
274 ggml_backend_sched_reserve(sched, reserve_graph);
275
276 // compute
277 graph = build_graph(sched); // the graph and its tensors are single-use in terms of allocation, multi-use in terms of computation
278 for (int i = 0; i < 10; ++i) {
279 ggml_backend_sched_graph_compute(sched, graph); // on the first iteration the graph is allocated automatically
280 }
281
282 // if there are graph inputs:
283 graph = build_graph(sched); // get a new graph that is not allocated (the metadata for the old graph is freed once ggml_free is called)
284 ggml_backend_sched_reset(sched); // clear the allocation of the previous graph
285 ggml_backend_sched_alloc_graph(sched, graph); // explicitly allocate the new graph but do not execute it
286 ggml_backend_tensor_set(input_tensor, ...); // copy data to the newly allocated graph tensors
287 ggml_backend_sched_graph_compute(sched, graph); // execute the graph
288
289 // as an alternative to the above it is also possible to assign the inputs to a dedicated context and
290 // allocate them statically via ggml_backend_alloc_ctx_tensors
291 }
292 */
293
294 typedef struct ggml_backend_sched * ggml_backend_sched_t;
295
296 // Evaluation callback for each node in the graph (set with ggml_backend_sched_set_eval_callback)
297 // when ask == true, the scheduler wants to know if the user wants to observe this node
298 // this allows the scheduler to batch nodes together in order to evaluate them in a single call
299 //
300 // when ask == false, the scheduler is passing the node tensor to the user for observation
301 // if the user returns false, the scheduler will cancel the graph compute
302 //
303 typedef bool (*ggml_backend_sched_eval_callback)(struct ggml_tensor * t, bool ask, void * user_data);
304
305 // Initialize a backend scheduler, backends with low index are given priority over backends with high index
306 GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel, bool op_offload);
307 GGML_API void ggml_backend_sched_free(ggml_backend_sched_t sched);
308
309 // Initialize backend buffers from a measure graph
310 GGML_API void ggml_backend_sched_reserve_size(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph, size_t * sizes);
311 GGML_API bool ggml_backend_sched_reserve(ggml_backend_sched_t sched, struct ggml_cgraph * measure_graph); // returns success
312
313 GGML_API int ggml_backend_sched_get_n_backends(ggml_backend_sched_t sched);
314 GGML_API ggml_backend_t ggml_backend_sched_get_backend(ggml_backend_sched_t sched, int i);
315
316 // Get the number of splits of the last graph
317 GGML_API int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched);
318 GGML_API int ggml_backend_sched_get_n_copies(ggml_backend_sched_t sched);
319
320 GGML_API ggml_backend_buffer_type_t ggml_backend_sched_get_buffer_type(ggml_backend_sched_t sched, ggml_backend_t backend);
321 GGML_API size_t ggml_backend_sched_get_buffer_size(ggml_backend_sched_t sched, ggml_backend_t backend);
322
323 GGML_API void ggml_backend_sched_set_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node, ggml_backend_t backend);
324 GGML_API ggml_backend_t ggml_backend_sched_get_tensor_backend(ggml_backend_sched_t sched, struct ggml_tensor * node);
325
326 // Split graph without allocating it
327 GGML_API void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
328
329 // Allocate and compute graph on the backend scheduler
330 GGML_API bool ggml_backend_sched_alloc_graph(ggml_backend_sched_t sched, struct ggml_cgraph * graph); // returns success
331 GGML_API enum ggml_status ggml_backend_sched_graph_compute(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
332 GGML_API enum ggml_status ggml_backend_sched_graph_compute_async(ggml_backend_sched_t sched, struct ggml_cgraph * graph);
333 GGML_API void ggml_backend_sched_synchronize(ggml_backend_sched_t sched);
334
335 // Reset all assignments and allocators - must be called before changing the node backends or allocating a new graph.
336 // This in effect deallocates all tensors that were previously allocated and leaves them with dangling pointers.
337 // The correct way to use this API is to discard the deallocated tensors and create new ones.
338 GGML_API void ggml_backend_sched_reset(ggml_backend_sched_t sched);
339
340 // Set a callback to be called for each resulting node during graph compute
341 GGML_API void ggml_backend_sched_set_eval_callback(ggml_backend_sched_t sched, ggml_backend_sched_eval_callback callback, void * user_data);
342
343 //
344 // Utils
345 //
346
347 struct ggml_backend_graph_copy {
348 ggml_backend_buffer_t buffer;
349 struct ggml_context * ctx_allocated;
350 struct ggml_context * ctx_unallocated;
351 struct ggml_cgraph * graph;
352 };
353
354 // Copy a graph to a different backend
355 GGML_API struct ggml_backend_graph_copy ggml_backend_graph_copy(ggml_backend_t backend, struct ggml_cgraph * graph);
356 GGML_API void ggml_backend_graph_copy_free(struct ggml_backend_graph_copy copy);
357
358 typedef bool (*ggml_backend_eval_callback)(int node_index, struct ggml_tensor * t1, struct ggml_tensor * t2, void * user_data);
359
360 // Compare the output of two backends
361 GGML_API bool ggml_backend_compare_graph_backend(ggml_backend_t backend1, ggml_backend_t backend2, struct ggml_cgraph * graph, ggml_backend_eval_callback callback, void * user_data, struct ggml_tensor const * const * test_nodes, size_t num_test_nodes);
362
363 // Tensor initialization
364 GGML_API enum ggml_status ggml_backend_tensor_alloc(ggml_backend_buffer_t buffer, struct ggml_tensor * tensor, void * addr);
365 GGML_API enum ggml_status ggml_backend_view_init(struct ggml_tensor * tensor);
366
367 // CPU buffer types are always available
368 GGML_API ggml_backend_buffer_t ggml_backend_cpu_buffer_from_ptr(void * ptr, size_t size);
369 GGML_API ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void);
370
371#ifdef __cplusplus
372}
373#endif