1<html>
2
3<head>
4 <meta charset="UTF-8">
5 <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1" />
6 <meta name="color-scheme" content="light dark">
7 <title>llama.cpp - chat</title>
8
9 <style>
10 body {
11 font-family: system-ui;
12 font-size: 90%;
13 }
14
15 #container {
16 margin: 0em auto;
17 display: flex;
18 flex-direction: column;
19 justify-content: space-between;
20 height: 100%;
21 }
22
23 main {
24 margin: 3px;
25 display: flex;
26 flex-direction: column;
27 justify-content: space-between;
28 gap: 1em;
29
30 flex-grow: 1;
31 overflow-y: auto;
32
33 border: 1px solid #ccc;
34 border-radius: 5px;
35 padding: 0.5em;
36 }
37
38 body {
39 max-width: 600px;
40 min-width: 300px;
41 line-height: 1.2;
42 margin: 0 auto;
43 padding: 0 0.5em;
44 }
45
46 p {
47 overflow-wrap: break-word;
48 word-wrap: break-word;
49 hyphens: auto;
50 margin-top: 0.5em;
51 margin-bottom: 0.5em;
52 }
53
54 #write form {
55 margin: 1em 0 0 0;
56 display: flex;
57 flex-direction: column;
58 gap: 0.5em;
59 align-items: stretch;
60 }
61
62 .right {
63 display: flex;
64 flex-direction: row;
65 gap: 0.5em;
66 justify-content: flex-end;
67 }
68
69 fieldset {
70 border: none;
71 padding: 0;
72 margin: 0;
73 }
74
75 fieldset.two {
76 display: grid;
77 grid-template: "a a";
78 gap: 1em;
79 }
80
81 fieldset.three {
82 display: grid;
83 grid-template: "a a a";
84 gap: 1em;
85 }
86
87 details {
88 border: 1px solid #aaa;
89 border-radius: 4px;
90 padding: 0.5em 0.5em 0;
91 margin-top: 0.5em;
92 }
93
94 summary {
95 font-weight: bold;
96 margin: -0.5em -0.5em 0;
97 padding: 0.5em;
98 cursor: pointer;
99 }
100
101 details[open] {
102 padding: 0.5em;
103 }
104
105 .prob-set {
106 padding: 0.3em;
107 border-bottom: 1px solid #ccc;
108 }
109
110 .popover-content {
111 position: absolute;
112 background-color: white;
113 padding: 0.2em;
114 box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
115 }
116
117 textarea {
118 padding: 5px;
119 flex-grow: 1;
120 width: 100%;
121 }
122
123 pre code {
124 display: block;
125 background-color: #222;
126 color: #ddd;
127 }
128
129 code {
130 font-family: monospace;
131 padding: 0.1em 0.3em;
132 border-radius: 3px;
133 }
134
135 fieldset label {
136 margin: 0.5em 0;
137 display: block;
138 }
139
140 fieldset label.slim {
141 margin: 0 0.5em;
142 display: inline;
143 }
144
145 header,
146 footer {
147 text-align: center;
148 }
149
150 footer {
151 font-size: 80%;
152 color: #888;
153 }
154
155 .mode-chat textarea[name=prompt] {
156 height: 4.5em;
157 }
158
159 .mode-completion textarea[name=prompt] {
160 height: 10em;
161 }
162
163 [contenteditable] {
164 display: inline-block;
165 white-space: pre-wrap;
166 outline: 0px solid transparent;
167 }
168
169 @keyframes loading-bg-wipe {
170 0% {
171 background-position: 0%;
172 }
173
174 100% {
175 background-position: 100%;
176 }
177 }
178
179 .loading {
180 --loading-color-1: #eeeeee00;
181 --loading-color-2: #eeeeeeff;
182 background-size: 50% 100%;
183 background-image: linear-gradient(90deg, var(--loading-color-1), var(--loading-color-2), var(--loading-color-1));
184 animation: loading-bg-wipe 2s linear infinite;
185 }
186
187 @media (prefers-color-scheme: dark) {
188 .loading {
189 --loading-color-1: #22222200;
190 --loading-color-2: #222222ff;
191 }
192
193 .popover-content {
194 background-color: black;
195 }
196 }
197 </style>
198
199 <script type="module">
200 import {
201 html, h, signal, effect, computed, render, useSignal, useEffect, useRef, Component
202 } from './index.js';
203
204 import { llama } from './completion.js';
205 import { SchemaConverter } from './json-schema-to-grammar.mjs';
206 let selected_image = false;
207 var slot_id = -1;
208
209 const session = signal({
210 prompt: "This is a conversation between User and Llama, a friendly chatbot. Llama is helpful, kind, honest, good at writing, and never fails to answer any requests immediately and with precision.",
211 template: "{{prompt}}\n\n{{history}}\n{{char}}:",
212 historyTemplate: "{{name}}: {{message}}",
213 transcript: [],
214 type: "chat", // "chat" | "completion"
215 char: "Llama",
216 user: "User",
217 image_selected: ''
218 })
219
220 const params = signal({
221 n_predict: 400,
222 temperature: 0.7,
223 repeat_last_n: 256, // 0 = disable penalty, -1 = context size
224 repeat_penalty: 1.18, // 1.0 = disabled
225 top_k: 40, // <= 0 to use vocab size
226 top_p: 0.95, // 1.0 = disabled
227 min_p: 0.05, // 0 = disabled
228 typical_p: 1.0, // 1.0 = disabled
229 presence_penalty: 0.0, // 0.0 = disabled
230 frequency_penalty: 0.0, // 0.0 = disabled
231 mirostat: 0, // 0/1/2
232 mirostat_tau: 5, // target entropy
233 mirostat_eta: 0.1, // learning rate
234 grammar: '',
235 n_probs: 0, // no completion_probabilities,
236 min_keep: 0, // min probs from each sampler,
237 image_data: [],
238 cache_prompt: true,
239 api_key: ''
240 })
241
242 /* START: Support for storing prompt templates and parameters in browsers LocalStorage */
243
244 const local_storage_storageKey = "llamacpp_server_local_storage";
245
246 function local_storage_setDataFromObject(tag, content) {
247 localStorage.setItem(local_storage_storageKey + '/' + tag, JSON.stringify(content));
248 }
249
250 function local_storage_setDataFromRawText(tag, content) {
251 localStorage.setItem(local_storage_storageKey + '/' + tag, content);
252 }
253
254 function local_storage_getDataAsObject(tag) {
255 const item = localStorage.getItem(local_storage_storageKey + '/' + tag);
256 if (!item) {
257 return null;
258 } else {
259 return JSON.parse(item);
260 }
261 }
262
263 function local_storage_getDataAsRawText(tag) {
264 const item = localStorage.getItem(local_storage_storageKey + '/' + tag);
265 if (!item) {
266 return null;
267 } else {
268 return item;
269 }
270 }
271
272 // create a container for user templates and settings
273
274 const savedUserTemplates = signal({})
275 const selectedUserTemplate = signal({ name: '', template: { session: {}, params: {} } })
276
277 // let's import locally saved templates and settings if there are any
278 // user templates and settings are stored in one object
279 // in form of { "templatename": "templatedata" } and { "settingstemplatename":"settingsdata" }
280
281 console.log('Importing saved templates')
282
283 let importedTemplates = local_storage_getDataAsObject('user_templates')
284
285 if (importedTemplates) {
286 // saved templates were successfully imported.
287
288 console.log('Processing saved templates and updating default template')
289 params.value = { ...params.value, image_data: [] };
290
291 //console.log(importedTemplates);
292 savedUserTemplates.value = importedTemplates;
293
294 //override default template
295 savedUserTemplates.value.default = { session: session.value, params: params.value }
296 local_storage_setDataFromObject('user_templates', savedUserTemplates.value)
297 } else {
298 // no saved templates detected.
299
300 console.log('Initializing LocalStorage and saving default template')
301
302 savedUserTemplates.value = { "default": { session: session.value, params: params.value } }
303 local_storage_setDataFromObject('user_templates', savedUserTemplates.value)
304 }
305
306 function userTemplateResetToDefault() {
307 console.log('Resetting template to default')
308 selectedUserTemplate.value.name = 'default';
309 selectedUserTemplate.value.data = savedUserTemplates.value['default'];
310 }
311
312 function userTemplateApply(t) {
313 session.value = t.data.session;
314 session.value = { ...session.value, image_selected: '' };
315 params.value = t.data.params;
316 params.value = { ...params.value, image_data: [] };
317 }
318
319 function userTemplateResetToDefaultAndApply() {
320 userTemplateResetToDefault()
321 userTemplateApply(selectedUserTemplate.value)
322 }
323
324 function userTemplateLoadAndApplyAutosaved() {
325 // get autosaved last used template
326 let lastUsedTemplate = local_storage_getDataAsObject('user_templates_last')
327
328 if (lastUsedTemplate) {
329
330 console.log('Autosaved template found, restoring')
331
332 selectedUserTemplate.value = lastUsedTemplate
333 }
334 else {
335
336 console.log('No autosaved template found, using default template')
337 // no autosaved last used template was found, so load from default.
338
339 userTemplateResetToDefault()
340 }
341
342 console.log('Applying template')
343 // and update internal data from templates
344
345 userTemplateApply(selectedUserTemplate.value)
346 }
347
348 //console.log(savedUserTemplates.value)
349 //console.log(selectedUserTemplate.value)
350
351 function userTemplateAutosave() {
352 console.log('Template Autosave...')
353 if (selectedUserTemplate.value.name == 'default') {
354 // we don't want to save over default template, so let's create a new one
355 let newTemplateName = 'UserTemplate-' + Date.now().toString()
356 let newTemplate = { 'name': newTemplateName, 'data': { 'session': session.value, 'params': params.value } }
357
358 console.log('Saving as ' + newTemplateName)
359
360 // save in the autosave slot
361 local_storage_setDataFromObject('user_templates_last', newTemplate)
362
363 // and load it back and apply
364 userTemplateLoadAndApplyAutosaved()
365 } else {
366 local_storage_setDataFromObject('user_templates_last', { 'name': selectedUserTemplate.value.name, 'data': { 'session': session.value, 'params': params.value } })
367 }
368 }
369
370 console.log('Checking for autosaved last used template')
371 userTemplateLoadAndApplyAutosaved()
372
373 /* END: Support for storing prompt templates and parameters in browsers LocalStorage */
374
375 const llamaStats = signal(null)
376 const controller = signal(null)
377
378 // currently generating a completion?
379 const generating = computed(() => controller.value != null)
380
381 // has the user started a chat?
382 const chatStarted = computed(() => session.value.transcript.length > 0)
383
384 const transcriptUpdate = (transcript) => {
385 session.value = {
386 ...session.value,
387 transcript
388 }
389 }
390
391 // simple template replace
392 const template = (str, extraSettings) => {
393 let settings = session.value;
394 if (extraSettings) {
395 settings = { ...settings, ...extraSettings };
396 }
397 return String(str).replaceAll(/\{\{(.*?)\}\}/g, (_, key) => template(settings[key]));
398 }
399
400 async function runLlama(prompt, llamaParams, char) {
401 const currentMessages = [];
402 const history = session.value.transcript;
403 if (controller.value) {
404 throw new Error("already running");
405 }
406 controller.value = new AbortController();
407 for await (const chunk of llama(prompt, llamaParams, { controller: controller.value, api_url: location.pathname.replace(/\/+$/, '') })) {
408 const data = chunk.data;
409
410 if (data.stop) {
411 while (
412 currentMessages.length > 0 &&
413 currentMessages[currentMessages.length - 1].content.match(/\n$/) != null
414 ) {
415 currentMessages.pop();
416 }
417 transcriptUpdate([...history, [char, currentMessages]])
418 console.log("Completion finished: '", currentMessages.map(msg => msg.content).join(''), "', summary: ", data);
419 } else {
420 currentMessages.push(data);
421 slot_id = data.slot_id;
422 if (selected_image && !data.multimodal) {
423 alert("The server was not compiled for multimodal or the model projector can't be loaded.");
424 return;
425 }
426 transcriptUpdate([...history, [char, currentMessages]])
427 }
428
429 if (data.timings) {
430 llamaStats.value = data;
431 }
432 }
433
434 controller.value = null;
435 }
436
437 // send message to server
438 const chat = async (msg) => {
439 if (controller.value) {
440 console.log('already running...');
441 return;
442 }
443
444 transcriptUpdate([...session.value.transcript, ["{{user}}", msg]])
445
446 let prompt = template(session.value.template, {
447 message: msg,
448 history: session.value.transcript.flatMap(
449 ([name, data]) =>
450 template(
451 session.value.historyTemplate,
452 {
453 name,
454 message: Array.isArray(data) ?
455 data.map(msg => msg.content).join('').replace(/^\s/, '') :
456 data,
457 }
458 )
459 ).join("\n"),
460 });
461 if (selected_image) {
462 prompt = `A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\nUSER:[img-10]${msg}\nASSISTANT:`;
463 }
464 await runLlama(prompt, {
465 ...params.value,
466 slot_id: slot_id,
467 stop: ["</s>", template("{{char}}:"), template("{{user}}:")],
468 }, "{{char}}");
469 }
470
471 const runCompletion = () => {
472 if (controller.value) {
473 console.log('already running...');
474 return;
475 }
476 const { prompt } = session.value;
477 transcriptUpdate([...session.value.transcript, ["", prompt]]);
478 runLlama(prompt, {
479 ...params.value,
480 slot_id: slot_id,
481 stop: [],
482 }, "").finally(() => {
483 session.value.prompt = session.value.transcript.map(([_, data]) =>
484 Array.isArray(data) ? data.map(msg => msg.content).join('') : data
485 ).join('');
486 session.value.transcript = [];
487 })
488 }
489
490 const stop = (e) => {
491 e.preventDefault();
492 if (controller.value) {
493 controller.value.abort();
494 controller.value = null;
495 }
496 }
497
498 const reset = (e) => {
499 stop(e);
500 transcriptUpdate([]);
501 }
502
503 const uploadImage = (e) => {
504 e.preventDefault();
505 document.getElementById("fileInput").click();
506 document.getElementById("fileInput").addEventListener("change", function (event) {
507 const selectedFile = event.target.files[0];
508 if (selectedFile) {
509 const reader = new FileReader();
510 reader.onload = function () {
511 const image_data = reader.result;
512 session.value = { ...session.value, image_selected: image_data };
513 params.value = {
514 ...params.value, image_data: [
515 { data: image_data.replace(/data:image\/[^;]+;base64,/, ''), id: 10 }]
516 }
517 };
518 selected_image = true;
519 reader.readAsDataURL(selectedFile);
520 }
521 });
522 }
523
524 function MessageInput() {
525 const message = useSignal("")
526
527 const submit = (e) => {
528 stop(e);
529 chat(message.value);
530 message.value = "";
531 }
532
533 const enterSubmits = (event) => {
534 if (event.which === 13 && !event.shiftKey) {
535 submit(event);
536 }
537 }
538
539 return html`
540 <form onsubmit=${submit}>
541 <div>
542 <textarea
543 className=${generating.value ? "loading" : null}
544 oninput=${(e) => message.value = e.target.value}
545 onkeypress=${enterSubmits}
546 placeholder="Say something..."
547 rows=2
548 type="text"
549 value="${message}"
550 />
551 </div>
552 <div class="right">
553 <button type="submit" disabled=${generating.value}>Send</button>
554 <button onclick=${uploadImage}>Upload Image</button>
555 <button onclick=${stop} disabled=${!generating.value}>Stop</button>
556 <button onclick=${reset}>Reset</button>
557 </div>
558 </form>
559 `
560 }
561
562 function CompletionControls() {
563 const submit = (e) => {
564 stop(e);
565 runCompletion();
566 }
567 return html`
568 <div>
569 <button onclick=${submit} type="button" disabled=${generating.value}>Start</button>
570 <button onclick=${stop} disabled=${!generating.value}>Stop</button>
571 <button onclick=${reset}>Reset</button>
572 </div>`;
573 }
574
575 const ChatLog = (props) => {
576 const messages = session.value.transcript;
577 const container = useRef(null)
578
579 useEffect(() => {
580 // scroll to bottom (if needed)
581 const parent = container.current.parentElement;
582 if (parent && parent.scrollHeight <= parent.scrollTop + parent.offsetHeight + 300) {
583 parent.scrollTo(0, parent.scrollHeight)
584 }
585 }, [messages])
586
587 const isCompletionMode = session.value.type === 'completion'
588 const chatLine = ([user, data], index) => {
589 let message
590 const isArrayMessage = Array.isArray(data)
591 if (params.value.n_probs > 0 && isArrayMessage) {
592 message = html`<${Probabilities} data=${data} />`
593 } else {
594 const text = isArrayMessage ?
595 data.map(msg => msg.content).join('').replace(/^\s+/, '') :
596 data;
597 message = isCompletionMode ?
598 text :
599 html`<${Markdownish} text=${template(text)} />`
600 }
601 if (user) {
602 return html`<p key=${index}><strong>${template(user)}:</strong> ${message}</p>`
603 } else {
604 return isCompletionMode ?
605 html`<span key=${index}>${message}</span>` :
606 html`<p key=${index}>${message}</p>`
607 }
608 };
609
610 const handleCompletionEdit = (e) => {
611 session.value.prompt = e.target.innerText;
612 session.value.transcript = [];
613 }
614
615 return html`
616 <div id="chat" ref=${container} key=${messages.length}>
617 <img style="width: 60%;${!session.value.image_selected ? `display: none;` : ``}" src="${session.value.image_selected}"/>
618 <span contenteditable=${isCompletionMode} ref=${container} oninput=${handleCompletionEdit}>
619 ${messages.flatMap(chatLine)}
620 </span>
621 </div>`;
622 };
623
624 const ConfigForm = (props) => {
625 const updateSession = (el) => session.value = { ...session.value, [el.target.name]: el.target.value }
626 const updateParams = (el) => params.value = { ...params.value, [el.target.name]: el.target.value }
627 const updateParamsFloat = (el) => params.value = { ...params.value, [el.target.name]: parseFloat(el.target.value) }
628 const updateParamsInt = (el) => params.value = { ...params.value, [el.target.name]: Math.floor(parseFloat(el.target.value)) }
629 const updateParamsBool = (el) => params.value = { ...params.value, [el.target.name]: el.target.checked }
630
631 const grammarJsonSchemaPropOrder = signal('')
632 const updateGrammarJsonSchemaPropOrder = (el) => grammarJsonSchemaPropOrder.value = el.target.value
633 const convertJSONSchemaGrammar = async () => {
634 try {
635 let schema = JSON.parse(params.value.grammar)
636 const converter = new SchemaConverter({
637 prop_order: grammarJsonSchemaPropOrder.value
638 .split(',')
639 .reduce((acc, cur, i) => ({ ...acc, [cur.trim()]: i }), {}),
640 allow_fetch: true,
641 })
642 schema = await converter.resolveRefs(schema, 'input')
643 converter.visit(schema, '')
644 params.value = {
645 ...params.value,
646 grammar: converter.formatGrammar(),
647 }
648 } catch (e) {
649 alert(`Convert failed: ${e.message}`)
650 }
651 }
652
653 const FloatField = ({ label, max, min, name, step, value }) => {
654 return html`
655 <div>
656 <label for="${name}">${label}</label>
657 <input type="range" id="${name}" min="${min}" max="${max}" step="${step}" name="${name}" value="${value}" oninput=${updateParamsFloat} />
658 <span>${value}</span>
659 </div>
660 `
661 };
662
663 const IntField = ({ label, max, min, name, value }) => {
664 return html`
665 <div>
666 <label for="${name}">${label}</label>
667 <input type="range" id="${name}" min="${min}" max="${max}" name="${name}" value="${value}" oninput=${updateParamsInt} />
668 <span>${value}</span>
669 </div>
670 `
671 };
672
673 const BoolField = ({ label, name, value }) => {
674 return html`
675 <div>
676 <label for="${name}">${label}</label>
677 <input type="checkbox" id="${name}" name="${name}" checked="${value}" onclick=${updateParamsBool} />
678 </div>
679 `
680 };
681
682 const userTemplateReset = (e) => {
683 e.preventDefault();
684 userTemplateResetToDefaultAndApply()
685 }
686
687 const UserTemplateResetButton = () => {
688 if (selectedUserTemplate.value.name == 'default') {
689 return html`
690 <button disabled>Using default template</button>
691 `
692 }
693
694 return html`
695 <button onclick=${userTemplateReset}>Reset all to default</button>
696 `
697 };
698
699 useEffect(() => {
700 // autosave template on every change
701 userTemplateAutosave()
702 }, [session.value, params.value])
703
704 const GrammarControl = () => (
705 html`
706 <div>
707 <label for="template">Grammar</label>
708 <textarea id="grammar" name="grammar" placeholder="Use gbnf or JSON Schema+convert" value="${params.value.grammar}" rows=4 oninput=${updateParams}/>
709 <input type="text" name="prop-order" placeholder="order: prop1,prop2,prop3" oninput=${updateGrammarJsonSchemaPropOrder} />
710 <button type="button" onclick=${convertJSONSchemaGrammar}>Convert JSON Schema</button>
711 </div>
712 `
713 );
714
715 const PromptControlFieldSet = () => (
716 html`
717 <fieldset>
718 <div>
719 <label htmlFor="prompt">Prompt</label>
720 <textarea type="text" name="prompt" value="${session.value.prompt}" oninput=${updateSession}/>
721 </div>
722 </fieldset>
723 `
724 );
725
726 const ChatConfigForm = () => (
727 html`
728 ${PromptControlFieldSet()}
729
730 <fieldset class="two">
731 <div>
732 <label for="user">User name</label>
733 <input type="text" name="user" value="${session.value.user}" oninput=${updateSession} />
734 </div>
735
736 <div>
737 <label for="bot">Bot name</label>
738 <input type="text" name="char" value="${session.value.char}" oninput=${updateSession} />
739 </div>
740 </fieldset>
741
742 <fieldset>
743 <div>
744 <label for="template">Prompt template</label>
745 <textarea id="template" name="template" value="${session.value.template}" rows=4 oninput=${updateSession}/>
746 </div>
747
748 <div>
749 <label for="template">Chat history template</label>
750 <textarea id="template" name="historyTemplate" value="${session.value.historyTemplate}" rows=1 oninput=${updateSession}/>
751 </div>
752 ${GrammarControl()}
753 </fieldset>
754 `
755 );
756
757 const CompletionConfigForm = () => (
758 html`
759 ${PromptControlFieldSet()}
760 <fieldset>${GrammarControl()}</fieldset>
761 `
762 );
763
764 return html`
765 <form>
766 <fieldset class="two">
767 <${UserTemplateResetButton}/>
768 <div>
769 <label class="slim"><input type="radio" name="type" value="chat" checked=${session.value.type === "chat"} oninput=${updateSession} /> Chat</label>
770 <label class="slim"><input type="radio" name="type" value="completion" checked=${session.value.type === "completion"} oninput=${updateSession} /> Completion</label>
771 </div>
772 </fieldset>
773
774 ${session.value.type === 'chat' ? ChatConfigForm() : CompletionConfigForm()}
775
776 <fieldset class="two">
777 ${IntField({ label: "Predictions", max: 2048, min: -1, name: "n_predict", value: params.value.n_predict })}
778 ${FloatField({ label: "Temperature", max: 2.0, min: 0.0, name: "temperature", step: 0.01, value: params.value.temperature })}
779 ${FloatField({ label: "Penalize repeat sequence", max: 2.0, min: 0.0, name: "repeat_penalty", step: 0.01, value: params.value.repeat_penalty })}
780 ${IntField({ label: "Consider N tokens for penalize", max: 2048, min: 0, name: "repeat_last_n", value: params.value.repeat_last_n })}
781 ${IntField({ label: "Top-K sampling", max: 100, min: -1, name: "top_k", value: params.value.top_k })}
782 ${FloatField({ label: "Top-P sampling", max: 1.0, min: 0.0, name: "top_p", step: 0.01, value: params.value.top_p })}
783 ${FloatField({ label: "Min-P sampling", max: 1.0, min: 0.0, name: "min_p", step: 0.01, value: params.value.min_p })}
784 </fieldset>
785 <details>
786 <summary>More options</summary>
787 <fieldset class="two">
788 ${FloatField({ label: "Typical P", max: 1.0, min: 0.0, name: "typical_p", step: 0.01, value: params.value.typical_p })}
789 ${FloatField({ label: "Presence penalty", max: 1.0, min: 0.0, name: "presence_penalty", step: 0.01, value: params.value.presence_penalty })}
790 ${FloatField({ label: "Frequency penalty", max: 1.0, min: 0.0, name: "frequency_penalty", step: 0.01, value: params.value.frequency_penalty })}
791 </fieldset>
792 <hr />
793 <fieldset class="three">
794 <div>
795 <label><input type="radio" name="mirostat" value="0" checked=${params.value.mirostat == 0} oninput=${updateParamsInt} /> no Mirostat</label>
796 <label><input type="radio" name="mirostat" value="1" checked=${params.value.mirostat == 1} oninput=${updateParamsInt} /> Mirostat v1</label>
797 <label><input type="radio" name="mirostat" value="2" checked=${params.value.mirostat == 2} oninput=${updateParamsInt} /> Mirostat v2</label>
798 </div>
799 ${FloatField({ label: "Mirostat tau", max: 10.0, min: 0.0, name: "mirostat_tau", step: 0.01, value: params.value.mirostat_tau })}
800 ${FloatField({ label: "Mirostat eta", max: 1.0, min: 0.0, name: "mirostat_eta", step: 0.01, value: params.value.mirostat_eta })}
801 </fieldset>
802 <fieldset>
803 ${IntField({ label: "Show Probabilities", max: 10, min: 0, name: "n_probs", value: params.value.n_probs })}
804 </fieldset>
805 <fieldset>
806 ${IntField({ label: "Min Probabilities from each Sampler", max: 10, min: 0, name: "min_keep", value: params.value.min_keep })}
807 </fieldset>
808 <fieldset>
809 <label for="api_key">API Key</label>
810 <input type="text" name="api_key" value="${params.value.api_key}" placeholder="Enter API key" oninput=${updateParams} />
811 </fieldset>
812 </details>
813 </form>
814 `
815 }
816
817 const probColor = (p) => {
818 const r = Math.floor(192 * (1 - p));
819 const g = Math.floor(192 * p);
820 return `rgba(${r},${g},0,0.3)`;
821 }
822
823 const Probabilities = (params) => {
824 return params.data.map(msg => {
825 const { completion_probabilities } = msg;
826 if (
827 !completion_probabilities ||
828 completion_probabilities.length === 0
829 ) return msg.content
830
831 if (completion_probabilities.length > 1) {
832 // Not for byte pair
833 if (completion_probabilities[0].content.startsWith('byte: \\')) return msg.content
834
835 const splitData = completion_probabilities.map(prob => ({
836 content: prob.content,
837 completion_probabilities: [prob]
838 }))
839 return html`<${Probabilities} data=${splitData} />`
840 }
841
842 const { probs, content } = completion_probabilities[0]
843 const found = probs.find(p => p.tok_str === msg.content)
844 const pColor = found ? probColor(found.prob) : 'transparent'
845
846 const popoverChildren = html`
847 <div class="prob-set">
848 ${probs.map((p, index) => {
849 return html`
850 <div
851 key=${index}
852 title=${`prob: ${p.prob}`}
853 style=${{
854 padding: '0.3em',
855 backgroundColor: p.tok_str === content ? probColor(p.prob) : 'transparent'
856 }}
857 >
858 <span>${p.tok_str}: </span>
859 <span>${Math.floor(p.prob * 100)}%</span>
860 </div>
861 `
862 })}
863 </div>
864 `
865
866 return html`
867 <${Popover} style=${{ backgroundColor: pColor }} popoverChildren=${popoverChildren}>
868 ${msg.content.match(/\n/gim) ? html`<br />` : msg.content}
869 </>
870 `
871 });
872 }
873
874 // poor mans markdown replacement
875 const Markdownish = (params) => {
876 const md = params.text
877 .replace(/&/g, '&')
878 .replace(/</g, '<')
879 .replace(/>/g, '>')
880 .replace(/^#{1,6} (.*)$/gim, '<h3>$1</h3>')
881 .replace(/\*\*(.*?)\*\*/g, '<strong>$1</strong>')
882 .replace(/__(.*?)__/g, '<strong>$1</strong>')
883 .replace(/\*(.*?)\*/g, '<em>$1</em>')
884 .replace(/_(.*?)_/g, '<em>$1</em>')
885 .replace(/```.*?\n([\s\S]*?)```/g, '<pre><code>$1</code></pre>')
886 .replace(/`(.*?)`/g, '<code>$1</code>')
887 .replace(/\n/gim, '<br />');
888 return html`<span dangerouslySetInnerHTML=${{ __html: md }} />`;
889 };
890
891 const ModelGenerationInfo = (params) => {
892 if (!llamaStats.value) {
893 return html`<span/>`
894 }
895 return html`
896 <span>
897 ${llamaStats.value.tokens_predicted} predicted, ${llamaStats.value.tokens_cached} cached, ${llamaStats.value.timings.predicted_per_token_ms.toFixed()}ms per token, ${llamaStats.value.timings.predicted_per_second.toFixed(2)} tokens per second
898 </span>
899 `
900 }
901
902 // simple popover impl
903 const Popover = (props) => {
904 const isOpen = useSignal(false);
905 const position = useSignal({ top: '0px', left: '0px' });
906 const buttonRef = useRef(null);
907 const popoverRef = useRef(null);
908
909 const togglePopover = () => {
910 if (buttonRef.current) {
911 const rect = buttonRef.current.getBoundingClientRect();
912 position.value = {
913 top: `${rect.bottom + window.scrollY}px`,
914 left: `${rect.left + window.scrollX}px`,
915 };
916 }
917 isOpen.value = !isOpen.value;
918 };
919
920 const handleClickOutside = (event) => {
921 if (popoverRef.current && !popoverRef.current.contains(event.target) && !buttonRef.current.contains(event.target)) {
922 isOpen.value = false;
923 }
924 };
925
926 useEffect(() => {
927 document.addEventListener('mousedown', handleClickOutside);
928 return () => {
929 document.removeEventListener('mousedown', handleClickOutside);
930 };
931 }, []);
932
933 return html`
934 <span style=${props.style} ref=${buttonRef} onClick=${togglePopover}>${props.children}</span>
935 ${isOpen.value && html`
936 <${Portal} into="#portal">
937 <div
938 ref=${popoverRef}
939 class="popover-content"
940 style=${{
941 top: position.value.top,
942 left: position.value.left,
943 }}
944 >
945 ${props.popoverChildren}
946 </div>
947 </${Portal}>
948 `}
949 `;
950 };
951
952 // Source: preact-portal (https://github.com/developit/preact-portal/blob/master/src/preact-portal.js)
953 /** Redirect rendering of descendants into the given CSS selector */
954 class Portal extends Component {
955 componentDidUpdate(props) {
956 for (let i in props) {
957 if (props[i] !== this.props[i]) {
958 return setTimeout(this.renderLayer);
959 }
960 }
961 }
962
963 componentDidMount() {
964 this.isMounted = true;
965 this.renderLayer = this.renderLayer.bind(this);
966 this.renderLayer();
967 }
968
969 componentWillUnmount() {
970 this.renderLayer(false);
971 this.isMounted = false;
972 if (this.remote && this.remote.parentNode) this.remote.parentNode.removeChild(this.remote);
973 }
974
975 findNode(node) {
976 return typeof node === 'string' ? document.querySelector(node) : node;
977 }
978
979 renderLayer(show = true) {
980 if (!this.isMounted) return;
981
982 // clean up old node if moving bases:
983 if (this.props.into !== this.intoPointer) {
984 this.intoPointer = this.props.into;
985 if (this.into && this.remote) {
986 this.remote = render(html`<${PortalProxy} />`, this.into, this.remote);
987 }
988 this.into = this.findNode(this.props.into);
989 }
990
991 this.remote = render(html`
992 <${PortalProxy} context=${this.context}>
993 ${show && this.props.children || null}
994 </${PortalProxy}>
995 `, this.into, this.remote);
996 }
997
998 render() {
999 return null;
1000 }
1001 }
1002 // high-order component that renders its first child if it exists.
1003 // used as a conditional rendering proxy.
1004 class PortalProxy extends Component {
1005 getChildContext() {
1006 return this.props.context;
1007 }
1008 render({ children }) {
1009 return children || null;
1010 }
1011 }
1012
1013 function App(props) {
1014 useEffect(() => {
1015 const query = new URLSearchParams(location.search).get("q");
1016 if (query) chat(query);
1017 }, []);
1018
1019 return html`
1020 <div class="mode-${session.value.type}">
1021 <header>
1022 <h1>llama.cpp</h1>
1023 </header>
1024
1025 <section id="write">
1026 <${session.value.type === 'chat' ? MessageInput : CompletionControls} />
1027 </section>
1028
1029 <main id="content">
1030 <${chatStarted.value ? ChatLog : ConfigForm} />
1031 </main>
1032
1033 <footer>
1034 <p><${ModelGenerationInfo} /></p>
1035 <p>Powered by <a href="https://github.com/ggml-org/llama.cpp">llama.cpp</a> and <a href="https://ggml.ai">ggml.ai</a>.</p>
1036 </footer>
1037 </div>
1038 `;
1039 }
1040
1041 render(h(App), document.querySelector('#container'));
1042 </script>
1043</head>
1044
1045<body>
1046 <div id="container">
1047 <input type="file" id="fileInput" accept="image/*" style="display: none;">
1048 </div>
1049 <div id="portal"></div>
1050</body>
1051
1052</html>