1<!DOCTYPE html>
2
3<html>
4
5<head>
6 <meta charset="UTF-8">
7 <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1" />
8 <meta name="color-scheme" content="light dark">
9 <title>llama.cpp - chat</title>
10
11 <link rel="icon" type="image/x-icon" href="favicon.ico">
12 <link rel="stylesheet" href="style.css">
13
14 <script type="module">
15 import {
16 html, h, signal, effect, computed, render, useSignal, useEffect, useRef, Component
17 } from './index.js';
18
19 import { llama } from './completion.js';
20 import { SchemaConverter } from './json-schema-to-grammar.mjs';
21 import { promptFormats } from './prompt-formats.js';
22 import { systemPrompts } from './system-prompts.js'; // multilingual is wip
23 let selected_image = false;
24 var slot_id = -1;
25
26 const session = signal({
27 prompt: "",
28 template: "{{prompt}}\n{{history}}{{char}}",
29 historyTemplate: "{{name}}: {{message}}\n",
30 transcript: [],
31 type: "chat", // "chat" | "completion"
32 char: "ASSISTANT",
33 user: "USER",
34 image_selected: ''
35 })
36
37 const params = signal({
38 n_predict: 358, // 358 is a nice number
39 temperature: 0.8, // adapt all following parameters to optimized min-p requierements. If for non-english, set to 0.6 or lower
40 repeat_last_n: 0, // 0 = disable penalty, -1 = context size
41 repeat_penalty: 1.0, // 1.0 = disabled
42 dry_multiplier: 0.0, // 0.0 = disabled, 0.8 works well
43 dry_base: 1.75, // 0.0 = disabled
44 dry_allowed_length: 2, // tokens extending repetitions beyond this receive penalty, 2 works well
45 dry_penalty_last_n: -1, // how many tokens to scan for repetitions (0 = disable penalty, -1 = context size)
46 top_k: 0, // <= 0 to use vocab size
47 top_p: 1.0, // 1.0 = disabled
48 min_p: 0.05, // 0 = disabled; recommended for non-english: ~ 0.4
49 xtc_probability: 0.0, // 0 = disabled;
50 xtc_threshold: 0.1, // > 0.5 disables XTC;
51 typical_p: 1.0, // 1.0 = disabled
52 presence_penalty: 0.0, // 0.0 = disabled
53 frequency_penalty: 0.0, // 0.0 = disabled
54 mirostat: 0, // 0/1/2
55 mirostat_tau: 5, // target entropy
56 mirostat_eta: 0.1, // learning rate
57 grammar: '',
58 n_probs: 0, // no completion_probabilities,
59 min_keep: 0, // min probs from each sampler,
60 image_data: [],
61 cache_prompt: true,
62 api_key: ''
63 })
64
65
66
67 /* START: Support for storing prompt templates and parameters in browser's LocalStorage */
68
69 const local_storage_storageKey = "llamacpp_server_local_storage";
70
71 function local_storage_setDataFromObject(tag, content) {
72 localStorage.setItem(local_storage_storageKey + '/' + tag, JSON.stringify(content));
73 }
74
75 function local_storage_setDataFromRawText(tag, content) {
76 localStorage.setItem(local_storage_storageKey + '/' + tag, content);
77 }
78
79 function local_storage_getDataAsObject(tag) {
80 const item = localStorage.getItem(local_storage_storageKey + '/' + tag);
81 if (!item) {
82 return null;
83 } else {
84 return JSON.parse(item);
85 }
86 }
87
88 function local_storage_getDataAsRawText(tag) {
89 const item = localStorage.getItem(local_storage_storageKey + '/' + tag);
90 if (!item) {
91 return null;
92 } else {
93 return item;
94 }
95 }
96
97 // create a container for user templates and settings
98
99 const savedUserTemplates = signal({})
100 const selectedUserTemplate = signal({ name: '', template: { session: {}, params: {} } })
101
102 // let's import locally saved templates and settings if there are any
103 // user templates and settings are stored in one object
104 // in form of { "templatename": "templatedata" } and { "settingstemplatename":"settingsdata" }
105
106 console.log('Importing saved templates')
107
108 let importedTemplates = local_storage_getDataAsObject('user_templates')
109
110 if (importedTemplates) {
111 // saved templates were successfuly imported.
112
113 console.log('Processing saved templates and updating default template')
114 params.value = { ...params.value, image_data: [] };
115
116 //console.log(importedTemplates);
117 savedUserTemplates.value = importedTemplates;
118
119 //override default template
120 savedUserTemplates.value.default = { session: session.value, params: params.value }
121 local_storage_setDataFromObject('user_templates', savedUserTemplates.value)
122 } else {
123 // no saved templates detected.
124
125 console.log('Initializing LocalStorage and saving default template')
126
127 savedUserTemplates.value = { "default": { session: session.value, params: params.value } }
128 local_storage_setDataFromObject('user_templates', savedUserTemplates.value)
129 }
130
131 function userTemplateResetToDefault() {
132 console.log('Reseting themplate to default')
133 selectedUserTemplate.value.name = 'default';
134 selectedUserTemplate.value.data = savedUserTemplates.value['default'];
135 }
136
137 function userTemplateApply(t) {
138 session.value = t.data.session;
139 session.value = { ...session.value, image_selected: '' };
140 params.value = t.data.params;
141 params.value = { ...params.value, image_data: [] };
142 }
143
144 function userTemplateResetToDefaultAndApply() {
145 userTemplateResetToDefault()
146 userTemplateApply(selectedUserTemplate.value)
147 }
148
149 function userTemplateLoadAndApplyAutosaved() {
150 // get autosaved last used template
151 let lastUsedTemplate = local_storage_getDataAsObject('user_templates_last')
152
153 if (lastUsedTemplate) {
154
155 console.log('Autosaved template found, restoring')
156
157 selectedUserTemplate.value = lastUsedTemplate
158 }
159 else {
160
161 console.log('No autosaved template found, using default template')
162 // no autosaved last used template was found, so load from default.
163
164 userTemplateResetToDefault()
165 }
166
167 console.log('Applying template')
168 // and update internal data from templates
169
170 userTemplateApply(selectedUserTemplate.value)
171 }
172
173 //console.log(savedUserTemplates.value)
174 //console.log(selectedUserTemplate.value)
175
176 function userTemplateAutosave() {
177 console.log('Template Autosave...')
178 if (selectedUserTemplate.value.name == 'default') {
179 // we don't want to save over default template, so let's create a new one
180 let newTemplateName = 'UserTemplate-' + Date.now().toString()
181 let newTemplate = { 'name': newTemplateName, 'data': { 'session': session.value, 'params': params.value } }
182
183 console.log('Saving as ' + newTemplateName)
184
185 // save in the autosave slot
186 local_storage_setDataFromObject('user_templates_last', newTemplate)
187
188 // and load it back and apply
189 userTemplateLoadAndApplyAutosaved()
190 } else {
191 local_storage_setDataFromObject('user_templates_last', { 'name': selectedUserTemplate.value.name, 'data': { 'session': session.value, 'params': params.value } })
192 }
193 }
194
195 console.log('Checking for autosaved last used template')
196 userTemplateLoadAndApplyAutosaved()
197
198 /* END: Support for storing prompt templates and parameters in browser's LocalStorage */
199
200 const llamaStats = signal(null)
201 const controller = signal(null)
202
203 // currently generating a completion?
204 const generating = computed(() => controller.value != null)
205
206 // has the user started a chat?
207 const chatStarted = computed(() => session.value.transcript.length > 0)
208
209 const transcriptUpdate = (transcript) => {
210 session.value = {
211 ...session.value,
212 transcript
213 }
214 }
215
216 // simple template replace
217 const template = (str, extraSettings) => {
218 let settings = session.value;
219 if (extraSettings) {
220 settings = { ...settings, ...extraSettings };
221 }
222 return String(str).replaceAll(/\{\{(.*?)\}\}/g, (_, key) => template(settings[key]));
223 }
224
225 async function runLlama(prompt, llamaParams, char) {
226 const currentMessages = [];
227 const history = session.value.transcript;
228 if (controller.value) {
229 throw new Error("already running");
230 }
231 controller.value = new AbortController();
232 for await (const chunk of llama(prompt, llamaParams, { controller: controller.value, api_url: new URL('.', document.baseURI).href })) {
233 const data = chunk.data;
234 if (data.stop) {
235 while (
236 currentMessages.length > 0 &&
237 currentMessages[currentMessages.length - 1].content.match(/\n$/) != null
238 ) {
239 currentMessages.pop();
240 }
241 transcriptUpdate([...history, [char, currentMessages]])
242 console.log("Completion finished: '", currentMessages.map(msg => msg.content).join(''), "', summary: ", data);
243 } else {
244 currentMessages.push(data);
245 slot_id = data.slot_id;
246 if (selected_image && !data.multimodal) {
247 alert("The server was not compiled for multimodal or the model projector can't be loaded."); return;
248 }
249 transcriptUpdate([...history, [char, currentMessages]])
250 }
251 if (data.timings) {
252 // llamaStats.value = data.timings;
253 llamaStats.value = data;
254 }
255 }
256 controller.value = null;
257 }
258
259 // send message to server
260 const chat = async (msg) => {
261 if (controller.value) {
262 console.log('already running...');
263 return;
264 }
265 // just in case (e.g. llama2)
266 const suffix = session.value.userMsgSuffix || "";
267 const prefix = session.value.userMsgPrefix || "";
268 const userMsg = prefix + msg + suffix;
269
270 transcriptUpdate([...session.value.transcript, ["{{user}}", userMsg]])
271
272 let prompt = template(session.value.template, {
273 message: msg,
274 history: session.value.transcript.flatMap(
275 ([name, data]) =>
276 template(
277 session.value.historyTemplate,
278 {
279 name,
280 message: Array.isArray(data) ?
281 data.map(msg => msg.content).join('').replace(/^\s/, '') :
282 data,
283 }
284 )
285 ).join(''),
286 });
287 if (selected_image) {
288 prompt = `A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\nUSER:[img-10]${msg}\nASSISTANT:`;
289 }
290 await runLlama(prompt, {
291 ...params.value,
292 slot_id: slot_id,
293 stop: ["</s>", "<|end|>", "<|eot_id|>", "<|end_of_text|>", "<|im_end|>", "<|EOT|>", "<|END_OF_TURN_TOKEN|>", "<|end_of_turn|>", "<|endoftext|>", template("{{char}}"), template("{{user}}")],
294 }, "{{char}}");
295 }
296
297 const runCompletion = () => {
298 if (controller.value) {
299 console.log('already running...');
300 return;
301 }
302 const { prompt } = session.value;
303 transcriptUpdate([...session.value.transcript, ["", prompt]]);
304 runLlama(prompt, {
305 ...params.value,
306 slot_id: slot_id,
307 stop: [],
308 }, "").finally(() => {
309 session.value.prompt = session.value.transcript.map(([_, data]) =>
310 Array.isArray(data) ? data.map(msg => msg.content).join('') : data
311 ).join('');
312 session.value.transcript = [];
313 })
314 }
315
316 const stop = (e) => {
317 e.preventDefault();
318 if (controller.value) {
319 controller.value.abort();
320 controller.value = null;
321 }
322 }
323
324 const reset = (e) => {
325 stop(e);
326 transcriptUpdate([]);
327 }
328
329 const uploadImage = (e) => {
330 e.preventDefault();
331 document.getElementById("fileInput").click();
332 document.getElementById("fileInput").addEventListener("change", function (event) {
333 const selectedFile = event.target.files[0];
334 if (selectedFile) {
335 const reader = new FileReader();
336 reader.onload = function () {
337 const image_data = reader.result;
338 session.value = { ...session.value, image_selected: image_data };
339 params.value = {
340 ...params.value, image_data: [
341 { data: image_data.replace(/data:image\/[^;]+;base64,/, ''), id: 10 }]
342 }
343 };
344 selected_image = true;
345 reader.readAsDataURL(selectedFile);
346 }
347 });
348 }
349
350 function MessageInput() {
351 const message = useSignal("")
352
353 const submit = (e) => {
354 stop(e);
355 chat(message.value);
356 message.value = "";
357 }
358
359 const enterSubmits = (event) => {
360 if (event.which === 13 && !event.shiftKey) {
361 submit(event);
362 }
363 }
364
365 return html`
366 <form onsubmit=${submit}>
367 <div class="chat-input-container">
368 <textarea
369 id="chat-input" placeholder="Say Something ... (Shift + Enter for new line)"
370 class="${generating.value ? 'loading' : null}"
371 oninput=${(e) => message.value = e.target.value}
372 onkeypress=${enterSubmits}
373 rows="2"
374 type="text"
375 value="${message}"
376 ></textarea>
377 </div>
378
379 <div class="right">
380 <button class="button-back" onclick=${reset}>Back</button>
381 <button onclick=${uploadImage}>Upload Image</button>
382 <button onclick=${stop} disabled=${!generating.value}>Stop</button>
383 <button type="submit" disabled=${generating.value}>Submit</button>
384 </div>
385 </form>
386 `
387 }
388
389 // the completion view needs some ux improvements
390 function CompletionControls() {
391 const submit = (e) => {
392 stop(e);
393 runCompletion();
394 }
395 return html`
396 <div class="right">
397 <button onclick=${submit} type="button" disabled=${generating.value}>Start</button>
398 <button onclick=${stop} disabled=${!generating.value}>Stop</button>
399 <button onclick=${reset}>Back</button>
400 </div>`;
401 }
402
403 const ChatLog = (props) => {
404 const messages = session.value.transcript;
405 const container = useRef(null)
406
407 useEffect(() => {
408 // scroll to bottom (if needed)
409 const parent = container.current.parentElement;
410 if (parent && parent.scrollHeight <= parent.scrollTop + parent.offsetHeight + 300) {
411 parent.scrollTo(0, parent.scrollHeight)
412 }
413 }, [messages])
414
415 const isCompletionMode = session.value.type === 'completion'
416 const chatLine = ([user, data], index) => {
417 let message
418 const isArrayMessage = Array.isArray(data)
419 if (params.value.n_probs > 0 && isArrayMessage) {
420 message = html`<${Probabilities} data=${data} />`
421 } else {
422 const text = isArrayMessage ?
423 data.map(msg => msg.content).join('') :
424 data;
425 message = isCompletionMode ?
426 text :
427 html`<${Markdownish} text=${template(text)} />`
428 }
429 if (user) {
430 return html`<p key=${index}><strong class="chat-id-color">${template(user)}</strong> ${message}</p>`
431 } else {
432 return isCompletionMode ?
433 html`<span key=${index}>${message}</span>` :
434 html`<p key=${index}>${message}</p>`
435 }
436 };
437
438 const handleCompletionEdit = (e) => {
439 session.value.prompt = e.target.innerText;
440 session.value.transcript = [];
441 }
442
443 return html`
444 <div id="chat" ref=${container} key=${messages.length}>
445 <img style="width: 60%;${!session.value.image_selected ? `display: none;` : ``}" src="${session.value.image_selected}"/>
446 <span contenteditable=${isCompletionMode} ref=${container} oninput=${handleCompletionEdit}>
447 ${messages.flatMap(chatLine)}
448 </span>
449 </div>`;
450 };
451
452
453
454///////////// UI Improvements /////////////
455//
456//
457const handleToggleChange = (e) => {
458 const isChecked = e.target.checked;
459 session.value = { ...session.value, type: isChecked ? 'completion' : 'chat' };
460 localStorage.setItem('toggleState', isChecked);
461}
462//
463const loadToggleState = () => {
464 const storedState = localStorage.getItem('toggleState');
465 if (storedState !== null) {
466 const isChecked = storedState === 'true';
467 document.getElementById('toggle').checked = isChecked;
468 session.value = { ...session.value, type: isChecked ? 'completion' : 'chat' };
469 }
470}
471//
472document.addEventListener('DOMContentLoaded', loadToggleState);
473//
474//
475// function to update the prompt format
476function updatePromptFormat(e) {
477 const promptFormat = e.target.value;
478 if (promptFormats.hasOwnProperty(promptFormat)) {
479 session.value = {
480 ...session.value,
481 ...promptFormats[promptFormat]
482 };
483 } else {
484 // Use vicuna as llama.cpp's default setting, since it's most common
485 session.value = {
486 ...session.value,
487 template: "{{prompt}}\n{{history}}{{char}}",
488 historyTemplate: "{{name}}: {{message}}\n",
489 char: "ASSISTANT",
490 user: "USER"
491 };
492 }
493 console.log('Updated session value:', session.value);
494}
495//
496//
497// function to update the prompt format from the selected one
498function updatePromptFormatFromDropdown(element) {
499 const promptFormat = element.getAttribute('data-value');
500 console.log('Selected prompt format:', promptFormat); // debugging
501 updatePromptFormat({ target: { value: promptFormat } });
502}
503//
504//
505// function that adds the event listers as soon as the element is available
506function addEventListenersWhenAvailable() {
507 var themeSelector = document.getElementById('theme-selector');
508 if (themeSelector) {
509 themeSelector.addEventListener('change', function(event) {
510 // event-handler-code...
511 });
512 // placeholder event listeners
513 } else {
514 // if the element is not there yet, wait ahead
515 requestAnimationFrame(addEventListenersWhenAvailable);
516 }
517}
518//
519//
520// begin with the check
521requestAnimationFrame(addEventListenersWhenAvailable);
522//
523//
524// avoid default and create new event object with value from data value attribute
525function handleDropdownSelection(e, promptFormat) {
526 e.preventDefault();
527 const customEvent = {
528 target: {
529 value: promptFormat
530 }
531 };
532 // call our updatePromptFormat-function
533 updatePromptFormat(customEvent);
534}
535//
536//
537// function to update the system message
538function updateSystemPrompt(e) {
539 const SystemPrompt = e.target.value;
540 if (systemPrompts.hasOwnProperty(SystemPrompt)) {
541 session.value = {
542 ...session.value,
543 prompt: systemPrompts[SystemPrompt].systemPrompt
544 };
545 }
546}
547//
548//
549///////////// UI Improvements /////////////
550
551
552
553
554const ConfigForm = (props) => {
555 const updateSession = (el) => session.value = { ...session.value, [el.target.name]: el.target.value }
556 const updateParams = (el) => params.value = { ...params.value, [el.target.name]: el.target.value }
557 const updateParamsFloat = (el) => params.value = { ...params.value, [el.target.name]: parseFloat(el.target.value) }
558 const updateParamsInt = (el) => params.value = { ...params.value, [el.target.name]: Math.floor(parseFloat(el.target.value)) }
559 const updateParamsBool = (el) => params.value = { ...params.value, [el.target.name]: el.target.checked }
560
561 const grammarJsonSchemaPropOrder = signal('')
562 const updateGrammarJsonSchemaPropOrder = (el) => grammarJsonSchemaPropOrder.value = el.target.value
563 const convertJSONSchemaGrammar = async () => {
564 try {
565 let schema = JSON.parse(params.value.grammar)
566 const converter = new SchemaConverter({
567 prop_order: grammarJsonSchemaPropOrder.value
568 .split(',')
569 .reduce((acc, cur, i) => ({ ...acc, [cur.trim()]: i }), {}),
570 allow_fetch: true,
571 })
572 schema = await converter.resolveRefs(schema, 'input')
573 converter.visit(schema, '')
574 params.value = {
575 ...params.value,
576 grammar: converter.formatGrammar(),
577 }
578 } catch (e) {
579 alert(`Convert failed: ${e.message}`)
580 }
581 }
582
583 const FloatField = ({ label, title, max, min, name, step, value }) => {
584return html`
585<div>
586 <label for="${name}"><span title="${title}">${label}</span></label>
587 <input type="range" id="${name}" min="${min}" max="${max}" step="${step}" name="${name}" value="${value}" oninput=${updateParamsFloat} title="${title}" />
588 <span id="${name}-value">${value}</span>
589</div>
590`
591};
592
593const IntField = ({ label, title, max, min, step, name, value }) => {
594return html`
595<div>
596 <label for="${name}"><span title="${title}">${label}</span></label>
597 <input type="range" id="${name}" min="${min}" max="${max}" step="${step}" name="${name}" value="${value}" oninput=${updateParamsInt} title="${title}" />
598 <span id="${name}-value">${value}</span>
599</div>
600`
601};
602
603const BoolField = ({ label, title, name, value }) => {
604return html`
605<div>
606 <label for="${name}"><span title="${title}">${label}</span></label>
607 <input type="checkbox" id="${name}" name="${name}" checked="${value}" onclick=${updateParamsBool} title="${title}" />
608</div>
609`
610};
611
612 const userTemplateReset = (e) => {
613 e.preventDefault();
614 userTemplateResetToDefaultAndApply()
615 }
616
617 const UserTemplateResetButton = () => {
618 if (selectedUserTemplate.value.name == 'default') {
619 return html`
620 <button class="reset-button" id="id_reset" onclick="${userTemplateReset}">Reset</button>
621 `
622 }
623
624 return html`
625 <div class="button-container">
626 <button class="reset-button" title="Caution: This resets the entire form." onclick="${userTemplateReset}">Reset</button>
627 </div>
628 `
629 };
630
631 useEffect(() => {
632 // autosave template on every change
633 userTemplateAutosave()
634 }, [session.value, params.value])
635
636 const GrammarControl = () => (
637 html`
638 <div>
639 <div class="grammar">
640 <label for="template"></label>
641 <textarea id="grammar" name="grammar" placeholder="Use GBNF or JSON Schema + Converter" value="${params.value.grammar}" rows=4 oninput=${updateParams}/>
642 </div>
643 <div class="grammar-columns">
644 <div class="json-schema-controls">
645 <input type="text" name="prop-order" placeholder="Order: prop1,prop2,prop3" oninput=${updateGrammarJsonSchemaPropOrder} />
646 <button type="button" class="button-grammar" onclick=${convertJSONSchemaGrammar}>Convert JSON Schema</button>
647 </div>
648 </div>
649 </div>
650 `
651 );
652
653 const PromptControlFieldSet = () => (
654 html`
655 <fieldset>
656 <div class="input-container">
657 <label for="prompt" class="input-label">System</label>
658 <textarea
659 id="prompt"
660 class="persistent-input"
661 name="prompt"
662 placeholder="[Note] The following models do not support System Prompts by design:\nโข OpenChat\nโข Orion\nโข Phi-3\nโข Starling\nโข Yi-...-Chat"
663 value="${session.value.prompt}"
664 oninput=${updateSession}
665 ></textarea>
666 </div>
667 </fieldset>
668 `
669 );
670
671 const ChatConfigForm = () => (
672 html`
673 <fieldset class="dropdowns">
674 <div>
675 <select id="promptFormat" name="promptFormat" onchange=${updatePromptFormat}>
676 <option value="default">Prompt Style</option>
677 <option value=""></option>
678 <optgroup label="Common Prompt-Styles">
679 <option value="alpaca">Alpaca</option>
680 <option value="chatml">ChatML</option>
681 <option value="commandr">Command R/+</option>
682 <option value="llama2">Llama 2</option>
683 <option value="llama3">Llama 3</option>
684 <option value="phi3">Phi-3</option>
685 <option value="openchat">OpenChat/Starling</option>
686 <option value="vicuna">Vicuna</option>
687 <option value=""></option>
688 </optgroup>
689 <optgroup label="More Prompt-Styles">
690 <option value="vicuna">Airoboros L2</option>
691 <option value="vicuna">BakLLaVA-1</option>
692 <option value="alpaca">Code Cherry Pop</option>
693 <option value="deepseekCoder">Deepseek Coder</option>
694 <option value="chatml">Dolphin Mistral</option>
695 <option value="chatml">evolvedSeeker 1.3B</option>
696 <option value="vicuna">Goliath 120B</option>
697 <option value="vicuna">Jordan</option>
698 <option value="vicuna">LLaVA</option>
699 <option value="chatml">Leo Hessianai</option>
700 <option value="vicuna">Leo Mistral</option>
701 <option value="vicuna">Marx</option>
702 <option value="med42">Med42</option>
703 <option value="alpaca">MetaMath</option>
704 <option value="llama2">Mistral Instruct</option>
705 <option value="chatml">Mistral 7B OpenOrca</option>
706 <option value="alpaca">MythoMax</option>
707 <option value="neuralchat">Neural Chat</option>
708 <option value="vicuna">Nous Capybara</option>
709 <option value="nousHermes">Nous Hermes</option>
710 <option value="openchatMath">OpenChat Math</option>
711 <option value="chatml">OpenHermes 2.5-Mistral</option>
712 <option value="alpaca">Orca Mini v3</option>
713 <option value="orion">Orion</option>
714 <option value="vicuna">Samantha</option>
715 <option value="chatml">Samantha Mistral</option>
716 <option value="sauerkrautLM">SauerkrautLM</option>
717 <option value="vicuna">Scarlett</option>
718 <option value="starlingCode">Starling Coding</option>
719 <option value="alpaca">Sydney</option>
720 <option value="vicuna">Synthia</option>
721 <option value="vicuna">Tess</option>
722 <option value="yi34b">Yi-6/9/34B-Chat</option>
723 <option value="zephyr">Zephyr</option>
724 <option value=""></option>
725 </optgroup>
726 </select>
727 <select id="SystemPrompt" name="SystemPrompt" onchange=${updateSystemPrompt}>
728 <option value="default">System Prompt</option>
729 <option value="empty">None</option>
730 <option value="airoboros">Airoboros</option>
731 <option value="alpaca">Alpaca</option>
732 <option value="atlas">Atlas</option>
733 <option value="atlas_de">Atlas - DE</option>
734 <option value="cot">Chain of Tought</option>
735 <option value="commandrempty">Command R/+ (empty)</option>
736 <option value="commandrexample">Command R/+ (example)</option>
737 <option value="deduce">Critical Thinking</option>
738 <option value="deepseekcoder">Deepseek Coder</option>
739 <option value="jordan">Jordan</option>
740 <option value="leomistral">Leo Mistral</option>
741 <option value="med42">Med42</option>
742 <option value="migeltot">Migel's Tree of Thought</option>
743 <option value="mistralopenorca">Mistral OpenOrca</option>
744 <option value="orcamini">Orca Mini</option>
745 <option value="samantha">Samantha</option>
746 <option value="sauerkraut">Sauerkraut</option>
747 <option value="scarlett">Scarlett</option>
748 <option value="synthia">Synthia</option>
749 <option value="vicuna">Vicuna</option>
750 </select>
751 <!--<select id="systemLanguage" name="systemLanguage">-->
752 <!--<option value="default">English</option>-->
753 <!--<option value="DE">German</option>-->
754 <!--<option value="placeholderLanguage">Placeholder</option>-->
755 <!--</select>-->
756 </div>
757 </fieldset>
758 ${PromptControlFieldSet()}
759 <fieldset>
760 <details>
761 <summary><span class="summary-title" id="id_prompt-style">Prompt Style</span></summary>
762 <fieldset class="names">
763 <div>
764 <label for="user" id="id_user-name">User ID</label>
765 <input type="text" id="user" name="user" value="${session.value.user}" oninput=${updateSession} />
766 </div>
767 <div>
768 <label for="bot" id="id_bot-name">AI ID</label>
769 <input type="text" id="bot" name="char" value="${session.value.char}" oninput=${updateSession} />
770 </div>
771 </fieldset>
772 <div class="two-columns">
773 <div>
774 <div class="input-container">
775 <label for="template" class="input-label-sec" id_prompt-template>Prompt Template</label>
776 <textarea id="template" class="persistent-input-sec" name="template" value="${session.value.template}" rows=6 oninput=${updateSession}/>
777 </div>
778 </div>
779 <div>
780 <div class="input-container">
781 <label for="template" class="input-label-sec" id="id_history-template">Chat History</label>
782 <textarea id="history-template" class="persistent-input-sec" name="historyTemplate" value="${session.value.historyTemplate}" rows=1 oninput=${updateSession}/>
783 </div>
784 </div>
785 </div>
786 </details>
787 <details>
788 <summary><span class="summary-title" id="id_grammar-title" id_grammar-title>Grammar</span></summary>
789 ${GrammarControl()}
790 </details>
791
792 </fieldset>
793 `
794 );
795
796 const CompletionConfigForm = () => (
797 html`
798 ${PromptControlFieldSet()}
799 <fieldset>
800 <details>
801 <summary><span class="summary-title" id="id_grammar-title" id_grammar-title>Grammar</span></summary>
802 ${GrammarControl()}
803 </details>
804 </fieldset>
805 `
806 );
807// todo toggle button et api field et reset button in one nice row
808 return html`
809 <form>
810 <fieldset class="two">
811 <input type="checkbox" id="toggle" class="toggleCheckbox" onchange=${handleToggleChange} />
812 <label for="toggle" class="toggleContainer">
813 <div id="id_toggle-label-chat">Chat</div>
814 <div id="id_toggle-label-complete">Complete</div>
815 </label>
816 <fieldset>
817
818 <input type="text" id="api_key" class="apiKey" name="api_key" value="${params.value.api_key}" placeholder="Enter API key" oninput=${updateParams} />
819 </fieldset>
820
821 <${UserTemplateResetButton}/>
822 </fieldset>
823
824 ${session.value.type === 'chat' ? ChatConfigForm() : CompletionConfigForm()}
825
826 <fieldset class="params">
827 ${IntField({ label: "Prediction", title: "Set the maximum number of tokens to predict when generating text. Note: May exceed the set limit slightly if the last token is a partial multibyte character. When 0, no tokens will be generated but the prompt is evaluated into the cache. The value -1 means infinity. Default is 358", max: 2048, min: -1, step: 16, name: "n_predict", value: params.value.n_predict, })}
828 ${FloatField({ label: "Min-P sampling", title: "The minimum probability for a token to be considered, relative to the probability of the most likely token. Note that it's good practice to disable all other samplers aside from temperature when using min-p. It is also recommenend to go this approach. Default is 0.05 โ But consider higher values like ~ 0.4 for non-English text generation. The value 1.0 means disabled", max: 1.0, min: 0.0, name: "min_p", step: 0.01, value: params.value.min_p })}
829 ${FloatField({ label: "Repetition Penalty", title: "Control the repetition of token sequences in the generated text. Default is 1.1", max: 2.0, min: 0.0, name: "repeat_penalty", step: 0.01, value: params.value.repeat_penalty })}
830 ${FloatField({ label: "Temperature", title: "This will adjust the overall randomness of the generated text. It is the most common sampler. Default is 0.8 but consider using lower values for more factual texts or for non-English text generation", max: 2.0, min: 0.0, name: "temperature", step: 0.01, value: params.value.temperature })}
831 </fieldset>
832
833 <details>
834 <summary><span class="summary-title">Further Options</span></summary>
835 <fieldset class="params">
836 ${IntField({ label: "Top-K", title: "Limits the selection of the next token to the K most probable tokens. 1 means no randomness = greedy sampling. If set to 0, it means the entire vocabulary size is considered.", max: 100, min: 0, step: 1, name: "top_k", value: params.value.top_k })}
837 ${IntField({ label: "Penalize Last N", title: "The last n tokens that are taken into account to penalise repetitions. A value of 0 means that this function is deactivated and -1 means that the entire size of the context is taken into account.", max: 2048, min: 0, step: 16, name: "repeat_last_n", value: params.value.repeat_last_n })}
838 ${FloatField({ label: "Presence Penalty", title: "A penalty that is applied if certain tokens appear repeatedly in the generated text. A higher value leads to fewer repetitions.", max: 1.0, min: 0.0, name: "presence_penalty", step: 0.01, value: params.value.presence_penalty })}
839 ${FloatField({ label: "Frequency Penalty", title: "A penalty that is applied based on the frequency with which certain tokens occur in the training data set. A higher value results in rare tokens being favoured.", max: 1.0, min: 0.0, name: "frequency_penalty", step: 0.01, value: params.value.frequency_penalty })}
840 ${FloatField({ label: "Top-P", title: "Limits the selection of the next token to a subset of tokens whose combined probability reaches a threshold value P = top-P. If set to 1, it means the entire vocabulary size is considered.", max: 1.0, min: 0.0, name: "top_p", step: 0.01, value: params.value.top_p })}
841 ${FloatField({ label: "Typical-P", title: "Activates local typical sampling, a method used to limit the prediction of tokens that are atypical in the current context. The parameter p controls the strength of this limitation. A value of 1.0 means that this function is deactivated.", max: 1.0, min: 0.0, name: "typical_p", step: 0.01, value: params.value.typical_p })}
842 ${FloatField({ label: "XTC probability", title: "Sets the chance for token removal (checked once on sampler start)", max: 1.0, min: 0.0, name: "xtc_probability", step: 0.01, value: params.value.xtc_probability })}
843 ${FloatField({ label: "XTC threshold", title: "Sets a minimum probability threshold for tokens to be removed", max: 0.5, min: 0.0, name: "xtc_threshold", step: 0.01, value: params.value.xtc_threshold })}
844 ${FloatField({ label: "DRY Penalty Multiplier", title: "Set the DRY repetition penalty multiplier. Default is 0.0, which disables DRY.", max: 5.0, min: 0.0, name: "dry_multiplier", step: 0.01, value: params.value.dry_multiplier })}
845 ${FloatField({ label: "DRY Base", title: "Set the DRY repetition penalty base value. Default is 1.75", max: 3.0, min: 1.0, name: "dry_base", step: 0.01, value: params.value.dry_base })}
846 ${IntField({ label: "DRY Allowed Length", title: "Tokens that extend repetition beyond this receive exponentially increasing penalty. Default is 2", max: 10, min: 1, step: 1, name: "dry_allowed_length", value: params.value.dry_allowed_length })}
847 ${IntField({ label: "DRY Penalty Last N", title: "How many tokens to scan for repetitions. Default is -1, where 0 is disabled and -1 is context size", max: 2048, min: -1, step: 16, name: "dry_penalty_last_n", value: params.value.dry_penalty_last_n })}
848 ${IntField({ label: "Min Keep", title: "If greater than 0, samplers are forced to return N possible tokens at minimum. Default is 0", max: 10, min: 0, name: "min_keep", value: params.value.min_keep })}
849 </fieldset>
850
851 <hr style="height: 1px; background-color: #ececf1; border: none;" />
852
853 <fieldset class="three">
854 <label title="The Mirostat sampling method is an algorithm used in natural language processing to improve the quality and coherence of the generated texts. It is an at-runtime-adaptive method that aims to keep the entropy or surprise of a text within a desired range."><input type="radio" name="mirostat" value="0" checked=${params.value.mirostat == 0} oninput=${updateParamsInt} /> Mirostat off</label>
855 <label title="Mirostat version 1 was developed to adjust the probability of predictions so that the surprise in the text remains constant. This means that the algorithm tries to maintain a balance between predictable and surprising words so that the text is neither too monotonous nor too chaotic. V1 is recommended for longer writings, creative texts, etc."><input type="radio" name="mirostat" value="1" checked=${params.value.mirostat == 1} oninput=${updateParamsInt} /> Mirostat v1</label>
856 <label title="Mirostat version 2 builds on the idea of V1 but brings some improvements. V2 is recommended as a general purpose algorithm since it offers more precise control over entropy and reacts more quickly to unwanted deviations. As a result, the generated texts appear even more consistent and coherent, especially for everday life conversations."><input type="radio" name="mirostat" value="2" checked=${params.value.mirostat == 2} oninput=${updateParamsInt} /> Mirostat v2</label>
857 </fieldset>
858 <fieldset class="params">
859 ${FloatField({ label: "Entropy tau", title: "Tau controls the desired level of entropy (or 'surprise') in the text. A low tau (e.g. 0.5) would mean that a text is very predictable, but will also be very coherent. A high tau (e.g. 8.0) would mean that the text is very creative and surprising, but may also be difficult to follow because unlikely words will occur frequently.", max: 10.0, min: 0.0, name: "mirostat_tau", step: 0.01, value: params.value.mirostat_tau })}
860 ${FloatField({ label: "Learning-rate eta", title: "Eta determines how quickly the Mirostat algorithm adjusts its predictions to achieve the desired entropy. A learning rate that is too high can cause the algorithm to react too quickly and possibly become unstable, because the algorithm will try to maintain a balance between surprises and precision in the context of only a few words. In this way, 'the common thread' could be lost. Whereas a learning rate that is too low means that the algorithm reacts too slowly and a red thread becomes a heavy goods train that takes a long time to come to a halt and change a 'topic station'.", max: 1.0, min: 0.0, name: "mirostat_eta", step: 0.01, value: params.value.mirostat_eta })}
861 </fieldset>
862
863 <hr style="height: 1px; background-color: #ececf1; border: none;" />
864
865 <fieldset class="params">
866 ${IntField({ label: "Show Probabilities", title: "If greater than 0, the response also contains the probabilities of top N tokens for each generated token given the sampling settings. The tokens will be colored in gradient from green to red depending on their probabilities. Note that for temperature 0 the tokens are sampled greedily but token probabilities are still being calculated via a simple softmax of the logits without considering any other sampler settings. Defaults to 0", max: 10, min: 0, step: 1, name: "n_probs", value: params.value.n_probs })}
867 </fieldset>
868 </details>
869 </form>
870 `
871}
872
873 // todo - beautify apikey section with css
874
875 const probColor = (p) => {
876 const r = Math.floor(192 * (1 - p));
877 const g = Math.floor(192 * p);
878 return `rgba(${r},${g},0,0.3)`;
879 }
880
881 const Probabilities = (params) => {
882 return params.data.map(msg => {
883 const { completion_probabilities } = msg;
884 if (
885 !completion_probabilities ||
886 completion_probabilities.length === 0
887 ) return msg.content
888
889 if (completion_probabilities.length > 1) {
890 // Not for byte pair
891 if (completion_probabilities[0].content.startsWith('byte: \\')) return msg.content
892
893 const splitData = completion_probabilities.map(prob => ({
894 content: prob.content,
895 completion_probabilities: [prob]
896 }))
897 return html`<${Probabilities} data=${splitData} />`
898 }
899
900 const { probs, content } = completion_probabilities[0]
901 const found = probs.find(p => p.tok_str === msg.content)
902 const pColor = found ? probColor(found.prob) : 'transparent'
903
904 const popoverChildren = html`
905 <div class="prob-set">
906 ${probs.map((p, index) => {
907 return html`
908 <div
909 key=${index}
910 title=${`prob: ${p.prob}`}
911 style=${{
912 padding: '0.3em',
913 backgroundColor: p.tok_str === content ? probColor(p.prob) : 'transparent'
914 }}
915 >
916 <span>${p.tok_str}: </span>
917 <span>${Math.floor(p.prob * 100)}%</span>
918 </div>
919 `
920 })}
921 </div>
922 `
923
924 return html`
925 <${Popover} style=${{ backgroundColor: pColor }} popoverChildren=${popoverChildren}>
926 ${msg.content.match(/\n/gim) ? html`<br />` : msg.content}
927 </>
928 `
929 });
930 }
931
932 // poor mans markdown replacement
933 const Markdownish = (params) => {
934 const md = params.text
935 .replace(/&/g, '&')
936 .replace(/</g, '<')
937 .replace(/>/g, '>')
938 .replace(/(^|\n)#{1,6} ([^\n]*)(?=([^`]*`[^`]*`)*[^`]*$)/g, '$1<h3>$2</h3>')
939 .replace(/\*\*(.*?)\*\*(?=([^`]*`[^`]*`)*[^`]*$)/g, '<strong>$1</strong>')
940 .replace(/__(.*?)__(?=([^`]*`[^`]*`)*[^`]*$)/g, '<strong>$1</strong>')
941 .replace(/\*(.*?)\*(?=([^`]*`[^`]*`)*[^`]*$)/g, '<em>$1</em>')
942 .replace(/_(.*?)_(?=([^`]*`[^`]*`)*[^`]*$)/g, '<em>$1</em>')
943 .replace(/```.*?\n([\s\S]*?)```/g, '<pre><code>$1</code></pre>')
944 .replace(/`(.*?)`/g, '<code>$1</code>')
945 .replace(/\n/gim, '<br />');
946 return html`<span dangerouslySetInnerHTML=${{ __html: md }} />`;
947 };
948
949 const ModelGenerationInfo = (params) => {
950 if (!llamaStats.value) {
951 return html`<span/>`
952 }
953 return html`
954 <span class=generation-statistics>
955 ${llamaStats.value.tokens_predicted} predicted, ${llamaStats.value.tokens_cached} cached, ${llamaStats.value.timings.predicted_per_second.toFixed(2)} tokens per second
956 </span>
957 `
958 }
959
960 // simple popover impl
961 const Popover = (props) => {
962 const isOpen = useSignal(false);
963 const position = useSignal({ top: '0px', left: '0px' });
964 const buttonRef = useRef(null);
965 const popoverRef = useRef(null);
966
967 const togglePopover = () => {
968 if (buttonRef.current) {
969 const rect = buttonRef.current.getBoundingClientRect();
970 position.value = {
971 top: `${rect.bottom + window.scrollY}px`,
972 left: `${rect.left + window.scrollX}px`,
973 };
974 }
975 isOpen.value = !isOpen.value;
976 };
977
978 const handleClickOutside = (event) => {
979 if (popoverRef.current && !popoverRef.current.contains(event.target) && !buttonRef.current.contains(event.target)) {
980 isOpen.value = false;
981 }
982 };
983
984 useEffect(() => {
985 document.addEventListener('mousedown', handleClickOutside);
986 return () => {
987 document.removeEventListener('mousedown', handleClickOutside);
988 };
989 }, []);
990
991 return html`
992 <span style=${props.style} ref=${buttonRef} onClick=${togglePopover} contenteditable="true">${props.children}</span>
993 ${isOpen.value && html`
994 <${Portal} into="#portal">
995 <div
996 ref=${popoverRef}
997 class="popover-content"
998 style=${{
999 top: position.value.top,
1000 left: position.value.left,
1001 }}
1002 >
1003 ${props.popoverChildren}
1004 </div>
1005 </${Portal}>
1006 `}
1007 `;
1008 };
1009
1010 // Source: preact-portal (https://github.com/developit/preact-portal/blob/master/src/preact-portal.js)
1011 /** Redirect rendering of descendants into the given CSS selector */
1012 class Portal extends Component {
1013 componentDidUpdate(props) {
1014 for (let i in props) {
1015 if (props[i] !== this.props[i]) {
1016 return setTimeout(this.renderLayer);
1017 }
1018 }
1019 }
1020
1021 componentDidMount() {
1022 this.isMounted = true;
1023 this.renderLayer = this.renderLayer.bind(this);
1024 this.renderLayer();
1025 }
1026
1027 componentWillUnmount() {
1028 this.renderLayer(false);
1029 this.isMounted = false;
1030 if (this.remote && this.remote.parentNode) this.remote.parentNode.removeChild(this.remote);
1031 }
1032
1033 findNode(node) {
1034 return typeof node === 'string' ? document.querySelector(node) : node;
1035 }
1036
1037 renderLayer(show = true) {
1038 if (!this.isMounted) return;
1039
1040 // clean up old node if moving bases:
1041 if (this.props.into !== this.intoPointer) {
1042 this.intoPointer = this.props.into;
1043 if (this.into && this.remote) {
1044 this.remote = render(html`<${PortalProxy} />`, this.into, this.remote);
1045 }
1046 this.into = this.findNode(this.props.into);
1047 }
1048
1049 this.remote = render(html`
1050 <${PortalProxy} context=${this.context}>
1051 ${show && this.props.children || null}
1052 </${PortalProxy}>
1053 `, this.into, this.remote);
1054 }
1055
1056 render() {
1057 return null;
1058 }
1059 }
1060 // high-order component that renders its first child if it exists.
1061 // used as a conditional rendering proxy.
1062 class PortalProxy extends Component {
1063 getChildContext() {
1064 return this.props.context;
1065 }
1066 render({ children }) {
1067 return children || null;
1068 }
1069 }
1070
1071 function App(props) {
1072 return html`
1073 <div class="mode-${session.value.type}">
1074 <header>
1075 <h2>llama.cpp</h2>
1076 <div class="dropdown">
1077 <button class="dropbtn"><svg width="16" height="16" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg"><circle cx="12" cy="12" r="10" stroke-width="2"/></svg></button>
1078 <div class="dropdown-content" id="theme-selector">
1079 <a href="/">Old UI</a>
1080 <a href="#" data-theme="default">Snow Storm</a>
1081 <a href="#" data-theme="polarnight">Polar Night</a>
1082 <a href="#" data-theme="ketivah">Ketivah</a>
1083 <a href="#" data-theme="mangotango">Mango Tango</a>
1084 <a href="#" data-theme="playground">Playground</a>
1085 <a href="#" data-theme="beeninorder">Been In Order</a>
1086 </div>
1087 </div>
1088 </header>
1089
1090 <main id="content">
1091 <${chatStarted.value ? ChatLog : ConfigForm} />
1092 </main>
1093
1094 <section id="write">
1095 <${session.value.type === 'chat' ? MessageInput : CompletionControls} />
1096 </section>
1097 <footer>
1098 <p><${ModelGenerationInfo} /></p>
1099 <p>Powered By <a href="https://github.com/ggml-org/llama.cpp#readme" target="_blank">llama.cpp</a> and <a href="https://ggml.ai/" target="_blank">ggml.ai</a></p>
1100 </footer>
1101 </div>
1102 `;
1103 }
1104
1105 document.addEventListener('DOMContentLoaded', function() {
1106 var themeSelector = document.getElementById('theme-selector');
1107 var themeLinks = themeSelector.querySelectorAll('a[data-theme]');
1108
1109 themeLinks.forEach(function(link) {
1110 link.addEventListener('click', function(event) {
1111 event.preventDefault(); // avoid default behaviour
1112 var selectedTheme = event.target.getAttribute('data-theme');
1113 changeTheme(selectedTheme);
1114 });
1115 });
1116
1117 function changeTheme(theme) {
1118 document.body.classList.remove('theme-default', 'theme-polarnight', 'theme-ketivah', 'theme-mangotango', 'theme-playground', 'theme-beeninorder');
1119 if (theme !== 'default') {
1120 document.body.classList.add('theme-' + theme);
1121 }
1122 localStorage.setItem('selected-theme', theme);
1123 }
1124
1125 // set the selected theme when loading the page
1126 var savedTheme = localStorage.getItem('selected-theme');
1127 if (savedTheme && savedTheme !== 'default') {
1128 document.body.classList.add('theme-' + savedTheme);
1129 // update the dropdown if it still exists
1130 var dropdown = document.getElementById('theme-selector-dropdown');
1131 if (dropdown) {
1132 dropdown.value = savedTheme;
1133 }
1134 }
1135});
1136
1137
1138// snapping of the slider to indicate 'disabled'
1139document.addEventListener('DOMContentLoaded', (event) => {
1140 // define an object that contains snap values and ranges for each slider
1141 const snapSettings = {
1142 temperature: { snapValue: 1.0, snapRangeMultiplier: 6 },
1143 min_p: { snapValue: 0.05, snapRangeMultiplier: 2 },
1144 xtc_probability: { snapValue: 0.0, snapRangeMultiplier: 4 },
1145 xtc_threshold: { snapValue: 0.5, snapRangeMultiplier: 4 },
1146 top_p: { snapValue: 1.0, snapRangeMultiplier: 4 },
1147 typical_p: { snapValue: 1.0, snapRangeMultiplier: 4 },
1148 repeat_penalty: { snapValue: 1.0, snapRangeMultiplier: 4 },
1149 presence_penalty: { snapValue: 0.0, snapRangeMultiplier: 4 },
1150 frequency_penalty: { snapValue: 0.0, snapRangeMultiplier: 4 },
1151 dry_multiplier: { snapValue: 0.0, snapRangeMultiplier: 4 },
1152 dry_base: { snapValue: 1.75, snapRangeMultiplier: 4 },
1153 };
1154 // add an event listener for each slider
1155 Object.keys(snapSettings).forEach(sliderName => {
1156 const slider = document.querySelector(`input[name="${sliderName}"]`);
1157 const settings = snapSettings[sliderName];
1158
1159 slider.addEventListener('input', (e) => {
1160 let value = parseFloat(e.target.value);
1161 const step = parseFloat(e.target.step);
1162 const snapRange = step * settings.snapRangeMultiplier;
1163 const valueDisplay = document.getElementById(`${e.target.name}-value`);
1164
1165 if (value >= settings.snapValue - snapRange && value <= settings.snapValue + snapRange) {
1166 value = settings.snapValue; // set value to the snap value
1167 e.target.value = value; // update the slider value
1168 }
1169 // update the displayed value
1170 if (valueDisplay) {
1171 valueDisplay.textContent = value.toFixed(2); // display value with two decimal places
1172 }
1173 });
1174 });
1175});
1176
1177 render(h(App), document.querySelector('#container'));
1178
1179 </script>
1180</head>
1181
1182<body>
1183
1184 <div id="container">
1185 <input type="file" id="fileInput" accept="image/*" style="display: none;">
1186 </div>
1187 <div id="portal"></div>
1188</body>
1189
1190</html>