1<html>
2<head>
3 <meta charset="UTF-8">
4 <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1" />
5 <meta name="color-scheme" content="light dark">
6 <title>llama.cpp - chat</title>
7
8 <style>
9 body {
10 font-family: system-ui;
11 font-size: 90%;
12 }
13
14 .grid-container {
15 display: grid;
16 grid-template-columns: auto auto auto;
17 padding: 10px;
18 }
19
20 .grid-item {
21 padding: 5px;
22 /* font-size: 30px; */
23 text-align: center;
24 }
25
26 #container {
27 margin: 0em auto;
28 display: flex;
29 flex-direction: column;
30 justify-content: space-between;
31 height: 100%;
32 }
33
34 main {
35 margin: 3px;
36 display: flex;
37 flex-direction: column;
38 justify-content: space-between;
39 gap: 1em;
40
41 flex-grow: 1;
42 overflow-y: auto;
43
44 border: 1px solid #ccc;
45 border-radius: 5px;
46 padding: 0.5em;
47 }
48
49 h1 {
50 text-align: center;
51 }
52
53 .customlink:link {
54 color: white;
55 background-color: #007aff;
56 font-weight: 600;
57 text-decoration: none;
58 float: right;
59 margin-top: 30px;
60 display: flex;
61 flex-direction: row;
62 gap: 0.5em;
63 justify-content: flex-end;
64 border-radius: 4px;
65 padding: 8px;
66 }
67
68 .customlink:visited {
69 color: white;
70 background-color: #007aff;
71 font-weight: 600;
72 text-decoration: none;
73 float: right;
74 margin-top: 30px;
75 display: flex;
76 flex-direction: row;
77 gap: 0.5em;
78 justify-content: flex-end;
79 padding: 8px;
80 }
81
82 .customlink:hover {
83 color: white;
84 background-color: #0070ee;
85 font-weight: 600;
86 text-decoration: none;
87 float: right;
88 margin-top: 30px;
89 display: flex;
90 flex-direction: row;
91 gap: 0.5em;
92 justify-content: flex-end;
93 padding: 8px;
94 }
95
96 .customlink:active {
97 color: #0070ee;
98 background-color: #80b3ef;
99 font-weight: 600;
100 text-decoration: none;
101 float: right;
102 margin-top: 30px;
103 display: flex;
104 flex-direction: row;
105 gap: 0.5em;
106 justify-content: flex-end;
107 padding: 8px;
108 }
109
110 body {
111 max-width: 600px;
112 min-width: 300px;
113 line-height: 1.2;
114 margin: 0 auto;
115 padding: 0 0.5em;
116 }
117
118 p {
119 overflow-wrap: break-word;
120 word-wrap: break-word;
121 hyphens: auto;
122 margin-top: 0.5em;
123 margin-bottom: 0.5em;
124 }
125
126 #write form {
127 margin: 1em 0 0 0;
128 display: flex;
129 flex-direction: column;
130 gap: 0.5em;
131 align-items: stretch;
132 }
133
134 .message-controls {
135 display: flex;
136 justify-content: flex-end;
137 }
138 .message-controls > div:nth-child(2) {
139 display: flex;
140 flex-direction: column;
141 gap: 0.5em;
142 }
143 .message-controls > div:nth-child(2) > div {
144 display: flex;
145 margin-left: auto;
146 gap: 0.5em;
147 }
148
149 fieldset {
150 border: none;
151 padding: 0;
152 margin: 0;
153 }
154
155 fieldset.two {
156 display: grid;
157 grid-template: "a a";
158 gap: 1em;
159 }
160
161 fieldset.three {
162 display: grid;
163 grid-template: "a a a";
164 gap: 1em;
165 }
166
167 details {
168 border: 1px solid #aaa;
169 border-radius: 4px;
170 padding: 0.5em 0.5em 0;
171 margin-top: 0.5em;
172 }
173
174 summary {
175 font-weight: bold;
176 margin: -0.5em -0.5em 0;
177 padding: 0.5em;
178 cursor: pointer;
179 }
180
181 details[open] {
182 padding: 0.5em;
183 }
184
185 .prob-set {
186 padding: 0.3em;
187 border-bottom: 1px solid #ccc;
188 }
189
190 .popover-content {
191 position: absolute;
192 background-color: white;
193 padding: 0.2em;
194 box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
195 }
196
197 textarea {
198 padding: 5px;
199 flex-grow: 1;
200 width: 100%;
201 }
202
203 pre code {
204 display: block;
205 background-color: #222;
206 color: #ddd;
207 }
208
209 code {
210 font-family: monospace;
211 padding: 0.1em 0.3em;
212 border-radius: 3px;
213 }
214
215 fieldset label {
216 margin: 0.5em 0;
217 display: block;
218 }
219
220 fieldset label.slim {
221 margin: 0 0.5em;
222 display: inline;
223 }
224
225 header,
226 footer {
227 text-align: center;
228 }
229
230 footer {
231 font-size: 80%;
232 color: #888;
233 }
234
235 .mode-chat textarea[name=prompt] {
236 height: 4.5em;
237 }
238
239 .mode-completion textarea[name=prompt] {
240 height: 10em;
241 }
242
243 [contenteditable] {
244 display: inline-block;
245 white-space: pre-wrap;
246 outline: 0px solid transparent;
247 }
248
249 @keyframes loading-bg-wipe {
250 0% {
251 background-position: 0%;
252 }
253
254 100% {
255 background-position: 100%;
256 }
257 }
258
259 .loading {
260 --loading-color-1: #eeeeee00;
261 --loading-color-2: #eeeeeeff;
262 background-size: 50% 100%;
263 background-image: linear-gradient(90deg, var(--loading-color-1), var(--loading-color-2), var(--loading-color-1));
264 animation: loading-bg-wipe 2s linear infinite;
265 }
266
267 @media (prefers-color-scheme: dark) {
268 .loading {
269 --loading-color-1: #22222200;
270 --loading-color-2: #222222ff;
271 }
272
273 .popover-content {
274 background-color: black;
275 }
276 }
277 </style>
278
279 <script type="module">
280 import {
281 html, h, signal, effect, computed, render, useSignal, useEffect, useRef, Component
282 } from './index.js';
283
284 import { llama } from './completion.js';
285 import { SchemaConverter } from './json-schema-to-grammar.mjs';
286
287 let selected_image = false;
288 var slot_id = -1;
289
290 const session = signal({
291 prompt: "This is a conversation between User and Llama, a friendly chatbot. Llama is helpful, kind, honest, good at writing, and never fails to answer any requests immediately and with precision.",
292 template: "{{prompt}}\n\n{{history}}\n{{char}}:",
293 historyTemplate: "{{name}}: {{message}}",
294 transcript: [],
295 type: "chat", // "chat" | "completion"
296 char: "Llama",
297 user: "User",
298 image_selected: ''
299 })
300
301 const params = signal({
302 n_predict: 400,
303 temperature: 0.7,
304 repeat_last_n: 256, // 0 = disable penalty, -1 = context size
305 repeat_penalty: 1.18, // 1.0 = disabled
306 dry_multiplier: 0.0, // 0.0 = disabled, 0.8 works well
307 dry_base: 1.75, // 0.0 = disabled
308 dry_allowed_length: 2, // tokens extending repetitions beyond this receive penalty, 2 works well
309 dry_penalty_last_n: -1, // how many tokens to scan for repetitions (0 = disable penalty, -1 = context size)
310 top_k: 40, // <= 0 to use vocab size
311 top_p: 0.95, // 1.0 = disabled
312 min_p: 0.05, // 0 = disabled
313 xtc_probability: 0.0, // 0 = disabled;
314 xtc_threshold: 0.1, // > 0.5 disables XTC;
315 typical_p: 1.0, // 1.0 = disabled
316 presence_penalty: 0.0, // 0.0 = disabled
317 frequency_penalty: 0.0, // 0.0 = disabled
318 mirostat: 0, // 0/1/2
319 mirostat_tau: 5, // target entropy
320 mirostat_eta: 0.1, // learning rate
321 grammar: '',
322 n_probs: 0, // no completion_probabilities,
323 min_keep: 0, // min probs from each sampler,
324 image_data: [],
325 cache_prompt: true,
326 api_key: ''
327 })
328
329 /* START: Support for storing prompt templates and parameters in browsers LocalStorage */
330
331 const local_storage_storageKey = "llamacpp_server_local_storage";
332
333 function local_storage_setDataFromObject(tag, content) {
334 localStorage.setItem(local_storage_storageKey + '/' + tag, JSON.stringify(content));
335 }
336
337 function local_storage_setDataFromRawText(tag, content) {
338 localStorage.setItem(local_storage_storageKey + '/' + tag, content);
339 }
340
341 function local_storage_getDataAsObject(tag) {
342 const item = localStorage.getItem(local_storage_storageKey + '/' + tag);
343 if (!item) {
344 return null;
345 } else {
346 return JSON.parse(item);
347 }
348 }
349
350 function local_storage_getDataAsRawText(tag) {
351 const item = localStorage.getItem(local_storage_storageKey + '/' + tag);
352 if (!item) {
353 return null;
354 } else {
355 return item;
356 }
357 }
358
359 // create a container for user templates and settings
360
361 const savedUserTemplates = signal({})
362 const selectedUserTemplate = signal({ name: '', template: { session: {}, params: {} } })
363
364 // let's import locally saved templates and settings if there are any
365 // user templates and settings are stored in one object
366 // in form of { "templatename": "templatedata" } and { "settingstemplatename":"settingsdata" }
367
368 console.log('Importing saved templates')
369
370 let importedTemplates = local_storage_getDataAsObject('user_templates')
371
372 if (importedTemplates) {
373 // saved templates were successfully imported.
374
375 console.log('Processing saved templates and updating default template')
376 params.value = { ...params.value, image_data: [] };
377
378 //console.log(importedTemplates);
379 savedUserTemplates.value = importedTemplates;
380
381 //override default template
382 savedUserTemplates.value.default = { session: session.value, params: params.value }
383 local_storage_setDataFromObject('user_templates', savedUserTemplates.value)
384 } else {
385 // no saved templates detected.
386
387 console.log('Initializing LocalStorage and saving default template')
388
389 savedUserTemplates.value = { "default": { session: session.value, params: params.value } }
390 local_storage_setDataFromObject('user_templates', savedUserTemplates.value)
391 }
392
393 function userTemplateResetToDefault() {
394 console.log('Resetting template to default')
395 selectedUserTemplate.value.name = 'default';
396 selectedUserTemplate.value.data = savedUserTemplates.value['default'];
397 }
398
399 function userTemplateApply(t) {
400 session.value = t.data.session;
401 session.value = { ...session.value, image_selected: '' };
402 params.value = t.data.params;
403 params.value = { ...params.value, image_data: [] };
404 }
405
406 function userTemplateResetToDefaultAndApply() {
407 userTemplateResetToDefault()
408 userTemplateApply(selectedUserTemplate.value)
409 }
410
411 function userTemplateLoadAndApplyAutosaved() {
412 // get autosaved last used template
413 let lastUsedTemplate = local_storage_getDataAsObject('user_templates_last')
414
415 if (lastUsedTemplate) {
416
417 console.log('Autosaved template found, restoring')
418
419 selectedUserTemplate.value = lastUsedTemplate
420 }
421 else {
422
423 console.log('No autosaved template found, using default template')
424 // no autosaved last used template was found, so load from default.
425
426 userTemplateResetToDefault()
427 }
428
429 console.log('Applying template')
430 // and update internal data from templates
431
432 userTemplateApply(selectedUserTemplate.value)
433 }
434
435 //console.log(savedUserTemplates.value)
436 //console.log(selectedUserTemplate.value)
437
438 function userTemplateAutosave() {
439 console.log('Template Autosave...')
440 if (selectedUserTemplate.value.name == 'default') {
441 // we don't want to save over default template, so let's create a new one
442 let newTemplateName = 'UserTemplate-' + Date.now().toString()
443 let newTemplate = { 'name': newTemplateName, 'data': { 'session': session.value, 'params': params.value } }
444
445 console.log('Saving as ' + newTemplateName)
446
447 // save in the autosave slot
448 local_storage_setDataFromObject('user_templates_last', newTemplate)
449
450 // and load it back and apply
451 userTemplateLoadAndApplyAutosaved()
452 } else {
453 local_storage_setDataFromObject('user_templates_last', { 'name': selectedUserTemplate.value.name, 'data': { 'session': session.value, 'params': params.value } })
454 }
455 }
456
457 console.log('Checking for autosaved last used template')
458 userTemplateLoadAndApplyAutosaved()
459
460 /* END: Support for storing prompt templates and parameters in browsers LocalStorage */
461
462 const tts = window.speechSynthesis;
463 const ttsVoice = signal(null)
464
465 const llamaStats = signal(null)
466 const controller = signal(null)
467
468 // currently generating a completion?
469 const generating = computed(() => controller.value != null)
470
471 // has the user started a chat?
472 const chatStarted = computed(() => session.value.transcript.length > 0)
473
474 const transcriptUpdate = (transcript) => {
475 session.value = {
476 ...session.value,
477 transcript
478 }
479 }
480
481 // simple template replace
482 const template = (str, extraSettings) => {
483 let settings = session.value;
484 if (extraSettings) {
485 settings = { ...settings, ...extraSettings };
486 }
487 return String(str).replaceAll(/\{\{(.*?)\}\}/g, (_, key) => template(settings[key]));
488 }
489
490 async function runLlama(prompt, llamaParams, char) {
491 const currentMessages = [];
492 const history = session.value.transcript;
493 if (controller.value) {
494 throw new Error("already running");
495 }
496 controller.value = new AbortController();
497 for await (const chunk of llama(prompt, llamaParams, { controller: controller.value, api_url: new URL('.', document.baseURI).href })) {
498 const data = chunk.data;
499
500 if (data.stop) {
501 while (
502 currentMessages.length > 0 &&
503 currentMessages[currentMessages.length - 1].content.match(/\n$/) != null
504 ) {
505 currentMessages.pop();
506 }
507 transcriptUpdate([...history, [char, currentMessages]])
508 console.log("Completion finished: '", currentMessages.map(msg => msg.content).join(''), "', summary: ", data);
509 } else {
510 currentMessages.push(data);
511 slot_id = data.slot_id;
512 if (selected_image && !data.multimodal) {
513 alert("The server was not compiled for multimodal or the model projector can't be loaded.");
514 return;
515 }
516 transcriptUpdate([...history, [char, currentMessages]])
517 }
518
519 if (data.timings) {
520 llamaStats.value = data;
521 }
522 }
523
524 controller.value = null;
525 }
526
527 // send message to server
528 const chat = async (msg) => {
529 if (controller.value) {
530 console.log('already running...');
531 return;
532 }
533
534 transcriptUpdate([...session.value.transcript, ["{{user}}", msg]])
535
536 let prompt = template(session.value.template, {
537 message: msg,
538 history: session.value.transcript.flatMap(
539 ([name, data]) =>
540 template(
541 session.value.historyTemplate,
542 {
543 name,
544 message: Array.isArray(data) ?
545 data.map(msg => msg.content).join('').replace(/^\s/, '') :
546 data,
547 }
548 )
549 ).join("\n"),
550 });
551 if (selected_image) {
552 prompt = `A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\nUSER:[img-10]${msg}\nASSISTANT:`;
553 }
554 await runLlama(prompt, {
555 ...params.value,
556 slot_id: slot_id,
557 stop: ["</s>", template("{{char}}:"), template("{{user}}:")],
558 }, "{{char}}");
559 }
560
561 const runCompletion = () => {
562 if (controller.value) {
563 console.log('already running...');
564 return;
565 }
566 const { prompt } = session.value;
567 transcriptUpdate([...session.value.transcript, ["", prompt]]);
568 runLlama(prompt, {
569 ...params.value,
570 slot_id: slot_id,
571 stop: [],
572 }, "").finally(() => {
573 session.value.prompt = session.value.transcript.map(([_, data]) =>
574 Array.isArray(data) ? data.map(msg => msg.content).join('') : data
575 ).join('');
576 session.value.transcript = [];
577 })
578 }
579
580 const stop = (e) => {
581 e.preventDefault();
582 if (controller.value) {
583 controller.value.abort();
584 controller.value = null;
585 }
586 }
587
588 const reset = (e) => {
589 stop(e);
590 transcriptUpdate([]);
591 }
592
593 const uploadImage = (e) => {
594 e.preventDefault();
595 document.getElementById("fileInput").click();
596 document.getElementById("fileInput").addEventListener("change", function (event) {
597 const selectedFile = event.target.files[0];
598 if (selectedFile) {
599 const reader = new FileReader();
600 reader.onload = function () {
601 const image_data = reader.result;
602 session.value = { ...session.value, image_selected: image_data };
603 params.value = {
604 ...params.value, image_data: [
605 { data: image_data.replace(/data:image\/[^;]+;base64,/, ''), id: 10 }]
606 }
607 };
608 selected_image = true;
609 reader.readAsDataURL(selectedFile);
610 }
611 });
612 }
613
614 const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
615 const talkRecognition = SpeechRecognition ? new SpeechRecognition() : null;
616 function MessageInput() {
617 const message = useSignal("");
618
619 const talkActive = useSignal(false);
620 const sendOnTalk = useSignal(false);
621 const talkStop = (e) => {
622 if (e) e.preventDefault();
623
624 talkActive.value = false;
625 talkRecognition?.stop();
626 }
627 const talk = (e) => {
628 e.preventDefault();
629
630 if (talkRecognition)
631 talkRecognition.start();
632 else
633 alert("Speech recognition is not supported by this browser.");
634 }
635 if(talkRecognition) {
636 talkRecognition.onstart = () => {
637 talkActive.value = true;
638 }
639 talkRecognition.onresult = (e) => {
640 if (event.results.length > 0) {
641 message.value = event.results[0][0].transcript;
642 if (sendOnTalk.value) {
643 submit(e);
644 }
645 }
646 }
647 talkRecognition.onspeechend = () => {
648 talkStop();
649 }
650 }
651
652 const ttsVoices = useSignal(tts?.getVoices() || []);
653 const ttsVoiceDefault = computed(() => ttsVoices.value.find(v => v.default));
654 if (tts) {
655 tts.onvoiceschanged = () => {
656 ttsVoices.value = tts.getVoices();
657 }
658 }
659
660 const submit = (e) => {
661 stop(e);
662 chat(message.value);
663 message.value = "";
664 }
665
666 const enterSubmits = (event) => {
667 if (event.which === 13 && !event.shiftKey) {
668 submit(event);
669 }
670 }
671
672 return html`
673 <form onsubmit=${submit}>
674 <div>
675 <textarea
676 className=${generating.value ? "loading" : null}
677 oninput=${(e) => message.value = e.target.value}
678 onkeypress=${enterSubmits}
679 placeholder="Say something..."
680 rows=2
681 type="text"
682 value="${message}"
683 />
684 </div>
685 <div class="message-controls">
686 <div> </div>
687 <div>
688 <div>
689 <button type="submit" disabled=${generating.value || talkActive.value}>Send</button>
690 <button disabled=${generating.value || talkActive.value} onclick=${uploadImage}>Upload Image</button>
691 <button onclick=${stop} disabled=${!generating.value}>Stop</button>
692 <button onclick=${reset}>Reset</button>
693 </div>
694 <div>
695 <a href="#" style="cursor: help;" title="Help" onclick=${e => {
696 e.preventDefault();
697 alert(`STT supported by your browser: ${SpeechRecognition ? 'Yes' : 'No'}\n` +
698 `(TTS and speech recognition are not provided by llama.cpp)\n` +
699 `Note: STT requires HTTPS to work.`);
700 }}>[?]</a>
701 <button disabled=${generating.value} onclick=${talkActive.value ? talkStop : talk}>${talkActive.value ? "Stop Talking" : "Talk"}</button>
702 <div>
703 <input type="checkbox" id="send-on-talk" name="send-on-talk" checked="${sendOnTalk}" onchange=${(e) => sendOnTalk.value = e.target.checked} />
704 <label for="send-on-talk" style="line-height: initial;">Send after talking</label>
705 </div>
706 </div>
707 <div>
708 <a href="#" style="cursor: help;" title="Help" onclick=${e => {
709 e.preventDefault();
710 alert(`TTS supported by your browser: ${tts ? 'Yes' : 'No'}\n(TTS and speech recognition are not provided by llama.cpp)`);
711 }}>[?]</a>
712 <label for="tts-voices" style="line-height: initial;">Bot Voice:</label>
713 <select id="tts-voices" name="tts-voices" onchange=${(e) => ttsVoice.value = e.target.value} style="max-width: 100px;">
714 <option value="" selected="${!ttsVoice.value}">None</option>
715 ${[
716 ...(ttsVoiceDefault.value ? [ttsVoiceDefault.value] : []),
717 ...ttsVoices.value.filter(v => !v.default),
718 ].map(
719 v => html`<option value="${v.name}" selected="${ttsVoice.value === v.name}">${v.name} (${v.lang}) ${v.default ? '(default)' : ''}</option>`
720 )}
721 </select>
722 </div>
723 </div>
724 </div>
725 </form>
726 `
727 }
728
729 function CompletionControls() {
730 const submit = (e) => {
731 stop(e);
732 runCompletion();
733 }
734 return html`
735 <div>
736 <button onclick=${submit} type="button" disabled=${generating.value}>Start</button>
737 <button onclick=${stop} disabled=${!generating.value}>Stop</button>
738 <button onclick=${reset}>Reset</button>
739 </div>`;
740 }
741
742 const ChatLog = (props) => {
743 const messages = session.value.transcript;
744 const container = useRef(null)
745
746 useEffect(() => {
747 // scroll to bottom (if needed)
748 const parent = container.current.parentElement;
749 if (parent && parent.scrollHeight <= parent.scrollTop + parent.offsetHeight + 300) {
750 parent.scrollTo(0, parent.scrollHeight)
751 }
752 }, [messages])
753
754 const ttsChatLineActiveIx = useSignal(undefined);
755 const ttsChatLine = (e, ix, msg) => {
756 if (e) e.preventDefault();
757
758 if (!tts || !ttsVoice.value || !('SpeechSynthesisUtterance' in window)) return;
759
760 const ttsVoices = tts.getVoices();
761 const voice = ttsVoices.find(v => v.name === ttsVoice.value);
762 if (!voice) return;
763
764 if (ttsChatLineActiveIx.value !== undefined) {
765 tts.cancel();
766 if (ttsChatLineActiveIx.value === ix) {
767 ttsChatLineActiveIx.value = undefined;
768 return;
769 }
770 }
771
772 ttsChatLineActiveIx.value = ix;
773 let ttsUtter = new SpeechSynthesisUtterance(msg);
774 ttsUtter.voice = voice;
775 ttsUtter.onend = e => {
776 ttsChatLineActiveIx.value = undefined;
777 };
778 tts.speak(ttsUtter);
779 }
780
781 const isCompletionMode = session.value.type === 'completion'
782
783 // Try play the last bot message
784 const lastCharChatLinesIxs = useSignal([]);
785 const lastCharChatLinesIxsOld = useSignal([]);
786 useEffect(() => {
787 if (
788 !isCompletionMode
789 && lastCharChatLinesIxs.value.length !== lastCharChatLinesIxsOld.value.length
790 && !generating.value
791 ) {
792 const ix = lastCharChatLinesIxs.value[lastCharChatLinesIxs.value.length - 1];
793 if (ix !== undefined) {
794 const msg = messages[ix];
795 ttsChatLine(null, ix, Array.isArray(msg) ? msg[1].map(m => m.content).join('') : msg);
796 }
797
798 lastCharChatLinesIxsOld.value = structuredClone(lastCharChatLinesIxs.value);
799 }
800 }, [generating.value]);
801
802 const chatLine = ([user, data], index) => {
803 let message
804 const isArrayMessage = Array.isArray(data);
805 const text = isArrayMessage ?
806 data.map(msg => msg.content).join('') :
807 data;
808 if (params.value.n_probs > 0 && isArrayMessage) {
809 message = html`<${Probabilities} data=${data} />`
810 } else {
811 message = isCompletionMode ?
812 text :
813 html`<${Markdownish} text=${template(text)} />`
814 }
815
816 const fromBot = user && user === '{{char}}';
817 if (fromBot && !lastCharChatLinesIxs.value.includes(index))
818 lastCharChatLinesIxs.value.push(index);
819
820 if (user) {
821 return html`
822 <div>
823 <p key=${index}><strong>${template(user)}:</strong> ${message}</p>
824 ${
825 fromBot && ttsVoice.value
826 && html`<button disabled=${generating.value} onclick=${e => ttsChatLine(e, index, text)} aria-label=${ttsChatLineActiveIx.value === index ? 'Pause' : 'Play'}>${ ttsChatLineActiveIx.value === index ? 'โธ๏ธ' : 'โถ๏ธ' }</div>`
827 }
828 </div>
829 `;
830 } else {
831 return isCompletionMode ?
832 html`<span key=${index}>${message}</span>` :
833 html`<div><p key=${index}>${message}</p></div>`
834 }
835 };
836
837 const handleCompletionEdit = (e) => {
838 session.value.prompt = e.target.innerText;
839 session.value.transcript = [];
840 }
841
842 return html`
843 <div id="chat" ref=${container} key=${messages.length}>
844 <img style="width: 60%;${!session.value.image_selected ? `display: none;` : ``}" src="${session.value.image_selected}"/>
845 <span contenteditable=${isCompletionMode} ref=${container} oninput=${handleCompletionEdit}>
846 ${messages.flatMap(chatLine)}
847 </span>
848 </div>`;
849 };
850
851 const ConfigForm = (props) => {
852 const updateSession = (el) => session.value = { ...session.value, [el.target.name]: el.target.value }
853 const updateParams = (el) => params.value = { ...params.value, [el.target.name]: el.target.value }
854 const updateParamsFloat = (el) => params.value = { ...params.value, [el.target.name]: parseFloat(el.target.value) }
855 const updateParamsInt = (el) => params.value = { ...params.value, [el.target.name]: Math.floor(parseFloat(el.target.value)) }
856 const updateParamsBool = (el) => params.value = { ...params.value, [el.target.name]: el.target.checked }
857
858 const grammarJsonSchemaPropOrder = signal('')
859 const updateGrammarJsonSchemaPropOrder = (el) => grammarJsonSchemaPropOrder.value = el.target.value
860 const convertJSONSchemaGrammar = async () => {
861 try {
862 let schema = JSON.parse(params.value.grammar)
863 const converter = new SchemaConverter({
864 prop_order: grammarJsonSchemaPropOrder.value
865 .split(',')
866 .reduce((acc, cur, i) => ({ ...acc, [cur.trim()]: i }), {}),
867 allow_fetch: true,
868 })
869 schema = await converter.resolveRefs(schema, 'input')
870 converter.visit(schema, '')
871 params.value = {
872 ...params.value,
873 grammar: converter.formatGrammar(),
874 }
875 } catch (e) {
876 alert(`Convert failed: ${e.message}`)
877 }
878 }
879
880 const FloatField = ({ label, max, min, name, step, value }) => {
881 return html`
882 <div>
883 <label for="${name}">${label}</label>
884 <input type="range" id="${name}" min="${min}" max="${max}" step="${step}" name="${name}" value="${value}" oninput=${updateParamsFloat} />
885 <span>${value}</span>
886 </div>
887 `
888 };
889
890 const IntField = ({ label, max, min, name, value }) => {
891 return html`
892 <div>
893 <label for="${name}">${label}</label>
894 <input type="range" id="${name}" min="${min}" max="${max}" name="${name}" value="${value}" oninput=${updateParamsInt} />
895 <span>${value}</span>
896 </div>
897 `
898 };
899
900 const BoolField = ({ label, name, value }) => {
901 return html`
902 <div>
903 <label for="${name}">${label}</label>
904 <input type="checkbox" id="${name}" name="${name}" checked="${value}" onclick=${updateParamsBool} />
905 </div>
906 `
907 };
908
909 const userTemplateReset = (e) => {
910 e.preventDefault();
911 userTemplateResetToDefaultAndApply()
912 }
913
914 const UserTemplateResetButton = () => {
915 if (selectedUserTemplate.value.name == 'default') {
916 return html`
917 <button disabled>Using default template</button>
918 `
919 }
920
921 return html`
922 <button onclick=${userTemplateReset}>Reset all to default</button>
923 `
924 };
925
926 useEffect(() => {
927 // autosave template on every change
928 userTemplateAutosave()
929 }, [session.value, params.value])
930
931 const GrammarControl = () => (
932 html`
933 <div>
934 <label for="template">Grammar</label>
935 <textarea id="grammar" name="grammar" placeholder="Use gbnf or JSON Schema+convert" value="${params.value.grammar}" rows=4 oninput=${updateParams}/>
936 <input type="text" name="prop-order" placeholder="order: prop1,prop2,prop3" oninput=${updateGrammarJsonSchemaPropOrder} />
937 <button type="button" onclick=${convertJSONSchemaGrammar}>Convert JSON Schema</button>
938 </div>
939 `
940 );
941
942 const PromptControlFieldSet = () => (
943 html`
944 <fieldset>
945 <div>
946 <label htmlFor="prompt">Prompt</label>
947 <textarea type="text" name="prompt" value="${session.value.prompt}" oninput=${updateSession}/>
948 </div>
949 </fieldset>
950 `
951 );
952
953 const ChatConfigForm = () => (
954 html`
955 ${PromptControlFieldSet()}
956
957 <fieldset class="two">
958 <div>
959 <label for="user">User name</label>
960 <input type="text" name="user" value="${session.value.user}" oninput=${updateSession} />
961 </div>
962
963 <div>
964 <label for="bot">Bot name</label>
965 <input type="text" name="char" value="${session.value.char}" oninput=${updateSession} />
966 </div>
967 </fieldset>
968
969 <fieldset>
970 <div>
971 <label for="template">Prompt template</label>
972 <textarea id="template" name="template" value="${session.value.template}" rows=4 oninput=${updateSession}/>
973 </div>
974
975 <div>
976 <label for="template">Chat history template</label>
977 <textarea id="template" name="historyTemplate" value="${session.value.historyTemplate}" rows=1 oninput=${updateSession}/>
978 </div>
979 ${GrammarControl()}
980 </fieldset>
981 `
982 );
983
984 const CompletionConfigForm = () => (
985 html`
986 ${PromptControlFieldSet()}
987 <fieldset>${GrammarControl()}</fieldset>
988 `
989 );
990
991 return html`
992 <form>
993 <fieldset class="two">
994 <${UserTemplateResetButton}/>
995 <div>
996 <label class="slim"><input type="radio" name="type" value="chat" checked=${session.value.type === "chat"} oninput=${updateSession} /> Chat</label>
997 <label class="slim"><input type="radio" name="type" value="completion" checked=${session.value.type === "completion"} oninput=${updateSession} /> Completion</label>
998 </div>
999 </fieldset>
1000
1001 ${session.value.type === 'chat' ? ChatConfigForm() : CompletionConfigForm()}
1002
1003 <fieldset class="two">
1004 ${IntField({ label: "Predictions", max: 2048, min: -1, name: "n_predict", value: params.value.n_predict })}
1005 ${FloatField({ label: "Temperature", max: 2.0, min: 0.0, name: "temperature", step: 0.01, value: params.value.temperature })}
1006 ${FloatField({ label: "Penalize repeat sequence", max: 2.0, min: 0.0, name: "repeat_penalty", step: 0.01, value: params.value.repeat_penalty })}
1007 ${IntField({ label: "Consider N tokens for penalize", max: 2048, min: 0, name: "repeat_last_n", value: params.value.repeat_last_n })}
1008 ${IntField({ label: "Top-K sampling", max: 100, min: -1, name: "top_k", value: params.value.top_k })}
1009 ${FloatField({ label: "Top-P sampling", max: 1.0, min: 0.0, name: "top_p", step: 0.01, value: params.value.top_p })}
1010 ${FloatField({ label: "Min-P sampling", max: 1.0, min: 0.0, name: "min_p", step: 0.01, value: params.value.min_p })}
1011 </fieldset>
1012 <details>
1013 <summary>More options</summary>
1014 <fieldset class="two">
1015 ${FloatField({ label: "Typical P", max: 1.0, min: 0.0, name: "typical_p", step: 0.01, value: params.value.typical_p })}
1016 ${FloatField({ label: "Presence penalty", max: 1.0, min: 0.0, name: "presence_penalty", step: 0.01, value: params.value.presence_penalty })}
1017 ${FloatField({ label: "Frequency penalty", max: 1.0, min: 0.0, name: "frequency_penalty", step: 0.01, value: params.value.frequency_penalty })}
1018 ${FloatField({ label: "DRY Penalty Multiplier", max: 5.0, min: 0.0, name: "dry_multiplier", step: 0.01, value: params.value.dry_multiplier })}
1019 ${FloatField({ label: "DRY Base", max: 3.0, min: 1.0, name: "dry_base", step: 0.01, value: params.value.dry_base })}
1020 ${IntField({ label: "DRY Allowed Length", max: 10, min: 2, step: 1, name: "dry_allowed_length", value: params.value.dry_allowed_length })}
1021 ${IntField({ label: "DRY Penalty Last N", max: 2048, min: -1, step: 16, name: "dry_penalty_last_n", value: params.value.dry_penalty_last_n })}
1022 ${FloatField({ label: "XTC probability", max: 1.0, min: 0.0, name: "xtc_probability", step: 0.01, value: params.value.xtc_probability })}
1023 ${FloatField({ label: "XTC threshold", max: 0.5, min: 0.0, name: "xtc_threshold", step: 0.01, value: params.value.xtc_threshold })}
1024 </fieldset>
1025 <hr />
1026 <fieldset class="three">
1027 <div>
1028 <label><input type="radio" name="mirostat" value="0" checked=${params.value.mirostat == 0} oninput=${updateParamsInt} /> no Mirostat</label>
1029 <label><input type="radio" name="mirostat" value="1" checked=${params.value.mirostat == 1} oninput=${updateParamsInt} /> Mirostat v1</label>
1030 <label><input type="radio" name="mirostat" value="2" checked=${params.value.mirostat == 2} oninput=${updateParamsInt} /> Mirostat v2</label>
1031 </div>
1032 ${FloatField({ label: "Mirostat tau", max: 10.0, min: 0.0, name: "mirostat_tau", step: 0.01, value: params.value.mirostat_tau })}
1033 ${FloatField({ label: "Mirostat eta", max: 1.0, min: 0.0, name: "mirostat_eta", step: 0.01, value: params.value.mirostat_eta })}
1034 </fieldset>
1035 <fieldset>
1036 ${IntField({ label: "Show Probabilities", max: 10, min: 0, name: "n_probs", value: params.value.n_probs })}
1037 </fieldset>
1038 <fieldset>
1039 ${IntField({ label: "Min Probabilities from each Sampler", max: 10, min: 0, name: "min_keep", value: params.value.min_keep })}
1040 </fieldset>
1041 <fieldset>
1042 <label for="api_key">API Key</label>
1043 <input type="text" name="api_key" value="${params.value.api_key}" placeholder="Enter API key" oninput=${updateParams} />
1044 </fieldset>
1045 </details>
1046 </form>
1047 `
1048 }
1049
1050 const probColor = (p) => {
1051 const r = Math.floor(192 * (1 - p));
1052 const g = Math.floor(192 * p);
1053 return `rgba(${r},${g},0,0.3)`;
1054 }
1055
1056 const Probabilities = (params) => {
1057 return params.data.map(msg => {
1058 const { completion_probabilities } = msg;
1059 if (
1060 !completion_probabilities ||
1061 completion_probabilities.length === 0
1062 ) return msg.content
1063
1064 if (completion_probabilities.length > 1) {
1065 // Not for byte pair
1066 if (completion_probabilities[0].content.startsWith('byte: \\')) return msg.content
1067
1068 const splitData = completion_probabilities.map(prob => ({
1069 content: prob.content,
1070 completion_probabilities: [prob]
1071 }))
1072 return html`<${Probabilities} data=${splitData} />`
1073 }
1074
1075 const { probs, content } = completion_probabilities[0]
1076 const found = probs.find(p => p.tok_str === msg.content)
1077 const pColor = found ? probColor(found.prob) : 'transparent'
1078
1079 const popoverChildren = html`
1080 <div class="prob-set">
1081 ${probs.map((p, index) => {
1082 return html`
1083 <div
1084 key=${index}
1085 title=${`prob: ${p.prob}`}
1086 style=${{
1087 padding: '0.3em',
1088 backgroundColor: p.tok_str === content ? probColor(p.prob) : 'transparent'
1089 }}
1090 >
1091 <span>${p.tok_str}: </span>
1092 <span>${Math.floor(p.prob * 100)}%</span>
1093 </div>
1094 `
1095 })}
1096 </div>
1097 `
1098
1099 return html`
1100 <${Popover} style=${{ backgroundColor: pColor }} popoverChildren=${popoverChildren}>
1101 ${msg.content.match(/\n/gim) ? html`<br />` : msg.content}
1102 </>
1103 `
1104 });
1105 }
1106
1107 // poor mans markdown replacement
1108 const Markdownish = (params) => {
1109 const chunks = params.text.split('```');
1110
1111 for (let i = 0; i < chunks.length; i++) {
1112 if (i % 2 === 0) { // outside code block
1113 chunks[i] = chunks[i]
1114 .replace(/&/g, '&')
1115 .replace(/</g, '<')
1116 .replace(/>/g, '>')
1117 .replace(/(^|\n)#{1,6} ([^\n]*)(?=([^`]*`[^`]*`)*[^`]*$)/g, '$1<h3>$2</h3>')
1118 .replace(/\*\*(.*?)\*\*(?=([^`]*`[^`]*`)*[^`]*$)/g, '<strong>$1</strong>')
1119 .replace(/__(.*?)__(?=([^`]*`[^`]*`)*[^`]*$)/g, '<strong>$1</strong>')
1120 .replace(/\*(.*?)\*(?=([^`]*`[^`]*`)*[^`]*$)/g, '<em>$1</em>')
1121 .replace(/_(.*?)_(?=([^`]*`[^`]*`)*[^`]*$)/g, '<em>$1</em>')
1122 .replace(/```.*?\n([\s\S]*?)```/g, '<pre><code>$1</code></pre>')
1123 .replace(/`(.*?)`/g, '<code>$1</code>')
1124 .replace(/\n/gim, '<br />');
1125 } else { // inside code block
1126 chunks[i] = `<pre><code>${chunks[i]}</code></pre>`;
1127 }
1128 }
1129
1130 const restoredText = chunks.join('');
1131
1132 return html`<span dangerouslySetInnerHTML=${{ __html: restoredText }} />`;
1133 };
1134
1135 const ModelGenerationInfo = (params) => {
1136 if (!llamaStats.value) {
1137 return html`<span/>`
1138 }
1139 return html`
1140 <span>
1141 ${llamaStats.value.tokens_predicted} predicted, ${llamaStats.value.tokens_cached} cached, ${llamaStats.value.timings.predicted_per_token_ms.toFixed()}ms per token, ${llamaStats.value.timings.predicted_per_second.toFixed(2)} tokens per second
1142 </span>
1143 `
1144 }
1145
1146
1147 // simple popover impl
1148 const Popover = (props) => {
1149 const isOpen = useSignal(false);
1150 const position = useSignal({ top: '0px', left: '0px' });
1151 const buttonRef = useRef(null);
1152 const popoverRef = useRef(null);
1153
1154 const togglePopover = () => {
1155 if (buttonRef.current) {
1156 const rect = buttonRef.current.getBoundingClientRect();
1157 position.value = {
1158 top: `${rect.bottom + window.scrollY}px`,
1159 left: `${rect.left + window.scrollX}px`,
1160 };
1161 }
1162 isOpen.value = !isOpen.value;
1163 };
1164
1165 const handleClickOutside = (event) => {
1166 if (popoverRef.current && !popoverRef.current.contains(event.target) && !buttonRef.current.contains(event.target)) {
1167 isOpen.value = false;
1168 }
1169 };
1170
1171 useEffect(() => {
1172 document.addEventListener('mousedown', handleClickOutside);
1173 return () => {
1174 document.removeEventListener('mousedown', handleClickOutside);
1175 };
1176 }, []);
1177
1178 return html`
1179 <span style=${props.style} ref=${buttonRef} onClick=${togglePopover}>${props.children}</span>
1180 ${isOpen.value && html`
1181 <${Portal} into="#portal">
1182 <div
1183 ref=${popoverRef}
1184 class="popover-content"
1185 style=${{
1186 top: position.value.top,
1187 left: position.value.left,
1188 }}
1189 >
1190 ${props.popoverChildren}
1191 </div>
1192 </${Portal}>
1193 `}
1194 `;
1195 };
1196
1197 // Source: preact-portal (https://github.com/developit/preact-portal/blob/master/src/preact-portal.js)
1198 /** Redirect rendering of descendants into the given CSS selector */
1199 class Portal extends Component {
1200 componentDidUpdate(props) {
1201 for (let i in props) {
1202 if (props[i] !== this.props[i]) {
1203 return setTimeout(this.renderLayer);
1204 }
1205 }
1206 }
1207
1208 componentDidMount() {
1209 this.isMounted = true;
1210 this.renderLayer = this.renderLayer.bind(this);
1211 this.renderLayer();
1212 }
1213
1214 componentWillUnmount() {
1215 this.renderLayer(false);
1216 this.isMounted = false;
1217 if (this.remote && this.remote.parentNode) this.remote.parentNode.removeChild(this.remote);
1218 }
1219
1220 findNode(node) {
1221 return typeof node === 'string' ? document.querySelector(node) : node;
1222 }
1223
1224 renderLayer(show = true) {
1225 if (!this.isMounted) return;
1226
1227 // clean up old node if moving bases:
1228 if (this.props.into !== this.intoPointer) {
1229 this.intoPointer = this.props.into;
1230 if (this.into && this.remote) {
1231 this.remote = render(html`<${PortalProxy} />`, this.into, this.remote);
1232 }
1233 this.into = this.findNode(this.props.into);
1234 }
1235
1236 this.remote = render(html`
1237 <${PortalProxy} context=${this.context}>
1238 ${show && this.props.children || null}
1239 </${PortalProxy}>
1240 `, this.into, this.remote);
1241 }
1242
1243 render() {
1244 return null;
1245 }
1246 }
1247 // high-order component that renders its first child if it exists.
1248 // used as a conditional rendering proxy.
1249 class PortalProxy extends Component {
1250 getChildContext() {
1251 return this.props.context;
1252 }
1253 render({ children }) {
1254 return children || null;
1255 }
1256 }
1257
1258 function App(props) {
1259 useEffect(() => {
1260 const query = new URLSearchParams(location.search).get("q");
1261 if (query) chat(query);
1262 }, []);
1263
1264 return html`
1265 <div class="mode-${session.value.type}">
1266 <header>
1267 <div class="grid-container">
1268 <div class="grid-item"></div>
1269 <div class="grid-item"><h1>llama.cpp</h1></div>
1270 <div class="grid-item"><a class="customlink" href="index-new.html">New UI</a></div>
1271 </div>
1272 </header>
1273
1274 <main id="content">
1275 <${chatStarted.value ? ChatLog : ConfigForm} />
1276 </main>
1277
1278 <section id="write">
1279 <${session.value.type === 'chat' ? MessageInput : CompletionControls} />
1280 </section>
1281
1282 <footer>
1283 <p><${ModelGenerationInfo} /></p>
1284 <p>Powered by <a href="https://github.com/ggml-org/llama.cpp">llama.cpp</a> and <a href="https://ggml.ai">ggml.ai</a>.</p>
1285 </footer>
1286 </div>
1287 `;
1288 }
1289
1290 render(h(App), document.querySelector('#container'));
1291 </script>
1292</head>
1293
1294<body>
1295 <div id="container">
1296 <input type="file" id="fileInput" accept="image/*" style="display: none;">
1297 </div>
1298 <div id="portal"></div>
1299</body>
1300
1301</html>