llmnpc - llama.cpp/tools/server/public

   1<html>
   2<head>
   3  <meta charset="UTF-8">
   4  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1" />
   5  <meta name="color-scheme" content="light dark">
   6  <title>llama.cpp - chat</title>
   7
   8  <style>
   9    body {
  10      font-family: system-ui;
  11      font-size: 90%;
  12    }
  13
  14    .grid-container {
  15      display: grid;
  16      grid-template-columns: auto auto auto;
  17      padding: 10px;
  18    }
  19
  20    .grid-item {
  21      padding: 5px;
  22      /* font-size: 30px; */
  23      text-align: center;
  24    }
  25
  26    #container {
  27      margin: 0em auto;
  28      display: flex;
  29      flex-direction: column;
  30      justify-content: space-between;
  31      height: 100%;
  32    }
  33
  34    main {
  35      margin: 3px;
  36      display: flex;
  37      flex-direction: column;
  38      justify-content: space-between;
  39      gap: 1em;
  40
  41      flex-grow: 1;
  42      overflow-y: auto;
  43
  44      border: 1px solid #ccc;
  45      border-radius: 5px;
  46      padding: 0.5em;
  47    }
  48
  49    h1 {
  50      text-align: center;
  51    }
  52
  53    .customlink:link {
  54      color: white;
  55      background-color: #007aff;
  56      font-weight: 600;
  57      text-decoration: none;
  58      float: right;
  59      margin-top: 30px;
  60      display: flex;
  61      flex-direction: row;
  62      gap: 0.5em;
  63      justify-content: flex-end;
  64      border-radius: 4px;
  65      padding: 8px;
  66    }
  67
  68    .customlink:visited {
  69      color: white;
  70      background-color: #007aff;
  71      font-weight: 600;
  72      text-decoration: none;
  73      float: right;
  74      margin-top: 30px;
  75      display: flex;
  76      flex-direction: row;
  77      gap: 0.5em;
  78      justify-content: flex-end;
  79      padding: 8px;
  80    }
  81
  82    .customlink:hover {
  83      color: white;
  84      background-color: #0070ee;
  85      font-weight: 600;
  86      text-decoration: none;
  87      float: right;
  88      margin-top: 30px;
  89      display: flex;
  90      flex-direction: row;
  91      gap: 0.5em;
  92      justify-content: flex-end;
  93      padding: 8px;
  94    }
  95
  96    .customlink:active {
  97      color: #0070ee;
  98      background-color: #80b3ef;
  99      font-weight: 600;
 100      text-decoration: none;
 101      float: right;
 102      margin-top: 30px;
 103      display: flex;
 104      flex-direction: row;
 105      gap: 0.5em;
 106      justify-content: flex-end;
 107      padding: 8px;
 108    }
 109
 110    body {
 111      max-width: 600px;
 112      min-width: 300px;
 113      line-height: 1.2;
 114      margin: 0 auto;
 115      padding: 0 0.5em;
 116    }
 117
 118    p {
 119      overflow-wrap: break-word;
 120      word-wrap: break-word;
 121      hyphens: auto;
 122      margin-top: 0.5em;
 123      margin-bottom: 0.5em;
 124    }
 125
 126    #write form {
 127      margin: 1em 0 0 0;
 128      display: flex;
 129      flex-direction: column;
 130      gap: 0.5em;
 131      align-items: stretch;
 132    }
 133
 134    .message-controls {
 135      display: flex;
 136      justify-content: flex-end;
 137    }
 138    .message-controls > div:nth-child(2) {
 139      display: flex;
 140      flex-direction: column;
 141      gap: 0.5em;
 142    }
 143    .message-controls > div:nth-child(2) > div {
 144      display: flex;
 145      margin-left: auto;
 146      gap: 0.5em;
 147    }
 148
 149    fieldset {
 150      border: none;
 151      padding: 0;
 152      margin: 0;
 153    }
 154
 155    fieldset.two {
 156      display: grid;
 157      grid-template: "a a";
 158      gap: 1em;
 159    }
 160
 161    fieldset.three {
 162      display: grid;
 163      grid-template: "a a a";
 164      gap: 1em;
 165    }
 166
 167    details {
 168      border: 1px solid #aaa;
 169      border-radius: 4px;
 170      padding: 0.5em 0.5em 0;
 171      margin-top: 0.5em;
 172    }
 173
 174    summary {
 175      font-weight: bold;
 176      margin: -0.5em -0.5em 0;
 177      padding: 0.5em;
 178      cursor: pointer;
 179    }
 180
 181    details[open] {
 182      padding: 0.5em;
 183    }
 184
 185    .prob-set {
 186      padding: 0.3em;
 187      border-bottom: 1px solid #ccc;
 188    }
 189
 190    .popover-content {
 191      position: absolute;
 192      background-color: white;
 193      padding: 0.2em;
 194      box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
 195    }
 196
 197    textarea {
 198      padding: 5px;
 199      flex-grow: 1;
 200      width: 100%;
 201    }
 202
 203    pre code {
 204      display: block;
 205      background-color: #222;
 206      color: #ddd;
 207    }
 208
 209    code {
 210      font-family: monospace;
 211      padding: 0.1em 0.3em;
 212      border-radius: 3px;
 213    }
 214
 215    fieldset label {
 216      margin: 0.5em 0;
 217      display: block;
 218    }
 219
 220    fieldset label.slim {
 221      margin: 0 0.5em;
 222      display: inline;
 223    }
 224
 225    header,
 226    footer {
 227      text-align: center;
 228    }
 229
 230    footer {
 231      font-size: 80%;
 232      color: #888;
 233    }
 234
 235    .mode-chat textarea[name=prompt] {
 236      height: 4.5em;
 237    }
 238
 239    .mode-completion textarea[name=prompt] {
 240      height: 10em;
 241    }
 242
 243    [contenteditable] {
 244      display: inline-block;
 245      white-space: pre-wrap;
 246      outline: 0px solid transparent;
 247    }
 248
 249    @keyframes loading-bg-wipe {
 250      0% {
 251        background-position: 0%;
 252      }
 253
 254      100% {
 255        background-position: 100%;
 256      }
 257    }
 258
 259    .loading {
 260      --loading-color-1: #eeeeee00;
 261      --loading-color-2: #eeeeeeff;
 262      background-size: 50% 100%;
 263      background-image: linear-gradient(90deg, var(--loading-color-1), var(--loading-color-2), var(--loading-color-1));
 264      animation: loading-bg-wipe 2s linear infinite;
 265    }
 266
 267    @media (prefers-color-scheme: dark) {
 268      .loading {
 269        --loading-color-1: #22222200;
 270        --loading-color-2: #222222ff;
 271      }
 272
 273      .popover-content {
 274        background-color: black;
 275      }
 276    }
 277  </style>
 278
 279  <script type="module">
 280    import {
 281      html, h, signal, effect, computed, render, useSignal, useEffect, useRef, Component
 282    } from './index.js';
 283
 284    import { llama } from './completion.js';
 285    import { SchemaConverter } from './json-schema-to-grammar.mjs';
 286
 287    let selected_image = false;
 288    var slot_id = -1;
 289
 290    const session = signal({
 291      prompt: "This is a conversation between User and Llama, a friendly chatbot. Llama is helpful, kind, honest, good at writing, and never fails to answer any requests immediately and with precision.",
 292      template: "{{prompt}}\n\n{{history}}\n{{char}}:",
 293      historyTemplate: "{{name}}: {{message}}",
 294      transcript: [],
 295      type: "chat",  // "chat" | "completion"
 296      char: "Llama",
 297      user: "User",
 298      image_selected: ''
 299    })
 300
 301    const params = signal({
 302      n_predict: 400,
 303      temperature: 0.7,
 304      repeat_last_n: 256, // 0 = disable penalty, -1 = context size
 305      repeat_penalty: 1.18, // 1.0 = disabled
 306      dry_multiplier: 0.0, // 0.0 = disabled, 0.8 works well
 307      dry_base: 1.75,     // 0.0 = disabled
 308      dry_allowed_length: 2, // tokens extending repetitions beyond this receive penalty, 2 works well
 309      dry_penalty_last_n: -1, // how many tokens to scan for repetitions (0 = disable penalty, -1 = context size)
 310      top_k: 40, // <= 0 to use vocab size
 311      top_p: 0.95, // 1.0 = disabled
 312      min_p: 0.05, // 0 = disabled
 313      xtc_probability: 0.0, // 0 = disabled;
 314      xtc_threshold: 0.1, // > 0.5 disables XTC;
 315      typical_p: 1.0, // 1.0 = disabled
 316      presence_penalty: 0.0, // 0.0 = disabled
 317      frequency_penalty: 0.0, // 0.0 = disabled
 318      mirostat: 0, // 0/1/2
 319      mirostat_tau: 5, // target entropy
 320      mirostat_eta: 0.1, // learning rate
 321      grammar: '',
 322      n_probs: 0, // no completion_probabilities,
 323      min_keep: 0, // min probs from each sampler,
 324      image_data: [],
 325      cache_prompt: true,
 326      api_key: ''
 327    })
 328
 329    /* START: Support for storing prompt templates and parameters in browsers LocalStorage */
 330
 331    const local_storage_storageKey = "llamacpp_server_local_storage";
 332
 333    function local_storage_setDataFromObject(tag, content) {
 334      localStorage.setItem(local_storage_storageKey + '/' + tag, JSON.stringify(content));
 335    }
 336
 337    function local_storage_setDataFromRawText(tag, content) {
 338      localStorage.setItem(local_storage_storageKey + '/' + tag, content);
 339    }
 340
 341    function local_storage_getDataAsObject(tag) {
 342      const item = localStorage.getItem(local_storage_storageKey + '/' + tag);
 343      if (!item) {
 344        return null;
 345      } else {
 346        return JSON.parse(item);
 347      }
 348    }
 349
 350    function local_storage_getDataAsRawText(tag) {
 351      const item = localStorage.getItem(local_storage_storageKey + '/' + tag);
 352      if (!item) {
 353        return null;
 354      } else {
 355        return item;
 356      }
 357    }
 358
 359    // create a container for user templates and settings
 360
 361    const savedUserTemplates = signal({})
 362    const selectedUserTemplate = signal({ name: '', template: { session: {}, params: {} } })
 363
 364    // let's import locally saved templates and settings if there are any
 365    // user templates and settings are stored in one object
 366    // in form of { "templatename": "templatedata" } and { "settingstemplatename":"settingsdata" }
 367
 368    console.log('Importing saved templates')
 369
 370    let importedTemplates = local_storage_getDataAsObject('user_templates')
 371
 372    if (importedTemplates) {
 373      // saved templates were successfully imported.
 374
 375      console.log('Processing saved templates and updating default template')
 376      params.value = { ...params.value, image_data: [] };
 377
 378      //console.log(importedTemplates);
 379      savedUserTemplates.value = importedTemplates;
 380
 381      //override default template
 382      savedUserTemplates.value.default = { session: session.value, params: params.value }
 383      local_storage_setDataFromObject('user_templates', savedUserTemplates.value)
 384    } else {
 385      // no saved templates detected.
 386
 387      console.log('Initializing LocalStorage and saving default template')
 388
 389      savedUserTemplates.value = { "default": { session: session.value, params: params.value } }
 390      local_storage_setDataFromObject('user_templates', savedUserTemplates.value)
 391    }
 392
 393    function userTemplateResetToDefault() {
 394      console.log('Resetting template to default')
 395      selectedUserTemplate.value.name = 'default';
 396      selectedUserTemplate.value.data = savedUserTemplates.value['default'];
 397    }
 398
 399    function userTemplateApply(t) {
 400      session.value = t.data.session;
 401      session.value = { ...session.value, image_selected: '' };
 402      params.value = t.data.params;
 403      params.value = { ...params.value, image_data: [] };
 404    }
 405
 406    function userTemplateResetToDefaultAndApply() {
 407      userTemplateResetToDefault()
 408      userTemplateApply(selectedUserTemplate.value)
 409    }
 410
 411    function userTemplateLoadAndApplyAutosaved() {
 412      // get autosaved last used template
 413      let lastUsedTemplate = local_storage_getDataAsObject('user_templates_last')
 414
 415      if (lastUsedTemplate) {
 416
 417        console.log('Autosaved template found, restoring')
 418
 419        selectedUserTemplate.value = lastUsedTemplate
 420      }
 421      else {
 422
 423        console.log('No autosaved template found, using default template')
 424        // no autosaved last used template was found, so load from default.
 425
 426        userTemplateResetToDefault()
 427      }
 428
 429      console.log('Applying template')
 430      // and update internal data from templates
 431
 432      userTemplateApply(selectedUserTemplate.value)
 433    }
 434
 435    //console.log(savedUserTemplates.value)
 436    //console.log(selectedUserTemplate.value)
 437
 438    function userTemplateAutosave() {
 439      console.log('Template Autosave...')
 440      if (selectedUserTemplate.value.name == 'default') {
 441        // we don't want to save over default template, so let's create a new one
 442        let newTemplateName = 'UserTemplate-' + Date.now().toString()
 443        let newTemplate = { 'name': newTemplateName, 'data': { 'session': session.value, 'params': params.value } }
 444
 445        console.log('Saving as ' + newTemplateName)
 446
 447        // save in the autosave slot
 448        local_storage_setDataFromObject('user_templates_last', newTemplate)
 449
 450        // and load it back and apply
 451        userTemplateLoadAndApplyAutosaved()
 452      } else {
 453        local_storage_setDataFromObject('user_templates_last', { 'name': selectedUserTemplate.value.name, 'data': { 'session': session.value, 'params': params.value } })
 454      }
 455    }
 456
 457    console.log('Checking for autosaved last used template')
 458    userTemplateLoadAndApplyAutosaved()
 459
 460    /* END: Support for storing prompt templates and parameters in browsers LocalStorage */
 461
 462    const tts = window.speechSynthesis;
 463    const ttsVoice = signal(null)
 464
 465    const llamaStats = signal(null)
 466    const controller = signal(null)
 467
 468    // currently generating a completion?
 469    const generating = computed(() => controller.value != null)
 470
 471    // has the user started a chat?
 472    const chatStarted = computed(() => session.value.transcript.length > 0)
 473
 474    const transcriptUpdate = (transcript) => {
 475      session.value = {
 476        ...session.value,
 477        transcript
 478      }
 479    }
 480
 481    // simple template replace
 482    const template = (str, extraSettings) => {
 483      let settings = session.value;
 484      if (extraSettings) {
 485        settings = { ...settings, ...extraSettings };
 486      }
 487      return String(str).replaceAll(/\{\{(.*?)\}\}/g, (_, key) => template(settings[key]));
 488    }
 489
 490    async function runLlama(prompt, llamaParams, char) {
 491      const currentMessages = [];
 492      const history = session.value.transcript;
 493      if (controller.value) {
 494        throw new Error("already running");
 495      }
 496      controller.value = new AbortController();
 497      for await (const chunk of llama(prompt, llamaParams, { controller: controller.value, api_url: new URL('.', document.baseURI).href })) {
 498        const data = chunk.data;
 499
 500        if (data.stop) {
 501          while (
 502            currentMessages.length > 0 &&
 503            currentMessages[currentMessages.length - 1].content.match(/\n$/) != null
 504          ) {
 505            currentMessages.pop();
 506          }
 507          transcriptUpdate([...history, [char, currentMessages]])
 508          console.log("Completion finished: '", currentMessages.map(msg => msg.content).join(''), "', summary: ", data);
 509        } else {
 510          currentMessages.push(data);
 511          slot_id = data.slot_id;
 512          if (selected_image && !data.multimodal) {
 513            alert("The server was not compiled for multimodal or the model projector can't be loaded.");
 514            return;
 515          }
 516          transcriptUpdate([...history, [char, currentMessages]])
 517        }
 518
 519        if (data.timings) {
 520          llamaStats.value = data;
 521        }
 522      }
 523
 524      controller.value = null;
 525    }
 526
 527    // send message to server
 528    const chat = async (msg) => {
 529      if (controller.value) {
 530        console.log('already running...');
 531        return;
 532      }
 533
 534      transcriptUpdate([...session.value.transcript, ["{{user}}", msg]])
 535
 536      let prompt = template(session.value.template, {
 537        message: msg,
 538        history: session.value.transcript.flatMap(
 539          ([name, data]) =>
 540            template(
 541              session.value.historyTemplate,
 542              {
 543                name,
 544                message: Array.isArray(data) ?
 545                  data.map(msg => msg.content).join('').replace(/^\s/, '') :
 546                  data,
 547              }
 548            )
 549        ).join("\n"),
 550      });
 551      if (selected_image) {
 552        prompt = `A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\nUSER:[img-10]${msg}\nASSISTANT:`;
 553      }
 554      await runLlama(prompt, {
 555        ...params.value,
 556        slot_id: slot_id,
 557        stop: ["</s>", template("{{char}}:"), template("{{user}}:")],
 558      }, "{{char}}");
 559    }
 560
 561    const runCompletion = () => {
 562      if (controller.value) {
 563        console.log('already running...');
 564        return;
 565      }
 566      const { prompt } = session.value;
 567      transcriptUpdate([...session.value.transcript, ["", prompt]]);
 568      runLlama(prompt, {
 569        ...params.value,
 570        slot_id: slot_id,
 571        stop: [],
 572      }, "").finally(() => {
 573        session.value.prompt = session.value.transcript.map(([_, data]) =>
 574          Array.isArray(data) ? data.map(msg => msg.content).join('') : data
 575        ).join('');
 576        session.value.transcript = [];
 577      })
 578    }
 579
 580    const stop = (e) => {
 581      e.preventDefault();
 582      if (controller.value) {
 583        controller.value.abort();
 584        controller.value = null;
 585      }
 586    }
 587
 588    const reset = (e) => {
 589      stop(e);
 590      transcriptUpdate([]);
 591    }
 592
 593    const uploadImage = (e) => {
 594      e.preventDefault();
 595      document.getElementById("fileInput").click();
 596      document.getElementById("fileInput").addEventListener("change", function (event) {
 597        const selectedFile = event.target.files[0];
 598        if (selectedFile) {
 599          const reader = new FileReader();
 600          reader.onload = function () {
 601            const image_data = reader.result;
 602            session.value = { ...session.value, image_selected: image_data };
 603            params.value = {
 604              ...params.value, image_data: [
 605                { data: image_data.replace(/data:image\/[^;]+;base64,/, ''), id: 10 }]
 606            }
 607          };
 608          selected_image = true;
 609          reader.readAsDataURL(selectedFile);
 610        }
 611      });
 612    }
 613
 614    const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
 615    const talkRecognition = SpeechRecognition ? new SpeechRecognition() : null;
 616    function MessageInput() {
 617      const message = useSignal("");
 618
 619      const talkActive = useSignal(false);
 620      const sendOnTalk = useSignal(false);
 621      const talkStop = (e) => {
 622        if (e) e.preventDefault();
 623
 624        talkActive.value = false;
 625        talkRecognition?.stop();
 626      }
 627      const talk = (e) => {
 628        e.preventDefault();
 629
 630        if (talkRecognition)
 631          talkRecognition.start();
 632        else
 633          alert("Speech recognition is not supported by this browser.");
 634      }
 635      if(talkRecognition) {
 636        talkRecognition.onstart = () => {
 637          talkActive.value = true;
 638        }
 639        talkRecognition.onresult = (e) => {
 640          if (event.results.length > 0) {
 641            message.value = event.results[0][0].transcript;
 642            if (sendOnTalk.value) {
 643              submit(e);
 644            }
 645          }
 646        }
 647        talkRecognition.onspeechend = () => {
 648          talkStop();
 649        }
 650      }
 651
 652      const ttsVoices = useSignal(tts?.getVoices() || []);
 653      const ttsVoiceDefault = computed(() => ttsVoices.value.find(v => v.default));
 654      if (tts) {
 655        tts.onvoiceschanged = () => {
 656          ttsVoices.value = tts.getVoices();
 657        }
 658      }
 659
 660      const submit = (e) => {
 661        stop(e);
 662        chat(message.value);
 663        message.value = "";
 664      }
 665
 666      const enterSubmits = (event) => {
 667        if (event.which === 13 && !event.shiftKey) {
 668          submit(event);
 669        }
 670      }
 671
 672      return html`
 673        <form onsubmit=${submit}>
 674          <div>
 675            <textarea
 676               className=${generating.value ? "loading" : null}
 677               oninput=${(e) => message.value = e.target.value}
 678               onkeypress=${enterSubmits}
 679               placeholder="Say something..."
 680               rows=2
 681               type="text"
 682               value="${message}"
 683            />
 684          </div>
 685          <div class="message-controls">
 686            <div> </div>
 687            <div>
 688              <div>
 689                <button type="submit" disabled=${generating.value || talkActive.value}>Send</button>
 690                <button disabled=${generating.value || talkActive.value} onclick=${uploadImage}>Upload Image</button>
 691                <button onclick=${stop} disabled=${!generating.value}>Stop</button>
 692                <button onclick=${reset}>Reset</button>
 693              </div>
 694              <div>
 695                <a href="#" style="cursor: help;" title="Help" onclick=${e => {
 696                  e.preventDefault();
 697                  alert(`STT supported by your browser: ${SpeechRecognition ? 'Yes' : 'No'}\n` +
 698                  `(TTS and speech recognition are not provided by llama.cpp)\n` +
 699                  `Note: STT requires HTTPS to work.`);
 700                }}>[?]</a>
 701                <button disabled=${generating.value} onclick=${talkActive.value ? talkStop : talk}>${talkActive.value ? "Stop Talking" : "Talk"}</button>
 702                <div>
 703                  <input type="checkbox" id="send-on-talk" name="send-on-talk" checked="${sendOnTalk}" onchange=${(e) => sendOnTalk.value = e.target.checked} />
 704                  <label for="send-on-talk" style="line-height: initial;">Send after talking</label>
 705                </div>
 706              </div>
 707              <div>
 708                <a href="#" style="cursor: help;" title="Help" onclick=${e => {
 709                  e.preventDefault();
 710                  alert(`TTS supported by your browser: ${tts ? 'Yes' : 'No'}\n(TTS and speech recognition are not provided by llama.cpp)`);
 711                }}>[?]</a>
 712                <label for="tts-voices" style="line-height: initial;">Bot Voice:</label>
 713                <select id="tts-voices" name="tts-voices" onchange=${(e) => ttsVoice.value = e.target.value} style="max-width: 100px;">
 714                  <option value="" selected="${!ttsVoice.value}">None</option>
 715                  ${[
 716                    ...(ttsVoiceDefault.value ? [ttsVoiceDefault.value] : []),
 717                    ...ttsVoices.value.filter(v => !v.default),
 718                  ].map(
 719                    v => html`<option value="${v.name}" selected="${ttsVoice.value === v.name}">${v.name} (${v.lang}) ${v.default ? '(default)' : ''}</option>`
 720                  )}
 721                </select>
 722              </div>
 723            </div>
 724          </div>
 725        </form>
 726      `
 727    }
 728
 729    function CompletionControls() {
 730      const submit = (e) => {
 731        stop(e);
 732        runCompletion();
 733      }
 734      return html`
 735        <div>
 736          <button onclick=${submit} type="button" disabled=${generating.value}>Start</button>
 737          <button onclick=${stop} disabled=${!generating.value}>Stop</button>
 738          <button onclick=${reset}>Reset</button>
 739        </div>`;
 740    }
 741
 742    const ChatLog = (props) => {
 743      const messages = session.value.transcript;
 744      const container = useRef(null)
 745
 746      useEffect(() => {
 747        // scroll to bottom (if needed)
 748        const parent = container.current.parentElement;
 749        if (parent && parent.scrollHeight <= parent.scrollTop + parent.offsetHeight + 300) {
 750          parent.scrollTo(0, parent.scrollHeight)
 751        }
 752      }, [messages])
 753
 754      const ttsChatLineActiveIx = useSignal(undefined);
 755      const ttsChatLine = (e, ix, msg) => {
 756        if (e) e.preventDefault();
 757
 758        if (!tts || !ttsVoice.value || !('SpeechSynthesisUtterance' in window)) return;
 759
 760        const ttsVoices = tts.getVoices();
 761        const voice = ttsVoices.find(v => v.name === ttsVoice.value);
 762        if (!voice) return;
 763
 764        if (ttsChatLineActiveIx.value !== undefined) {
 765          tts.cancel();
 766          if (ttsChatLineActiveIx.value === ix) {
 767            ttsChatLineActiveIx.value = undefined;
 768            return;
 769          }
 770        }
 771
 772        ttsChatLineActiveIx.value = ix;
 773        let ttsUtter = new SpeechSynthesisUtterance(msg);
 774        ttsUtter.voice = voice;
 775        ttsUtter.onend = e => {
 776          ttsChatLineActiveIx.value = undefined;
 777        };
 778        tts.speak(ttsUtter);
 779      }
 780
 781      const isCompletionMode = session.value.type === 'completion'
 782
 783      // Try play the last bot message
 784      const lastCharChatLinesIxs = useSignal([]);
 785      const lastCharChatLinesIxsOld = useSignal([]);
 786      useEffect(() => {
 787        if (
 788          !isCompletionMode
 789          && lastCharChatLinesIxs.value.length !== lastCharChatLinesIxsOld.value.length
 790          && !generating.value
 791        ) {
 792          const ix = lastCharChatLinesIxs.value[lastCharChatLinesIxs.value.length - 1];
 793          if (ix !== undefined) {
 794            const msg = messages[ix];
 795            ttsChatLine(null, ix, Array.isArray(msg) ? msg[1].map(m => m.content).join('') : msg);
 796          }
 797
 798          lastCharChatLinesIxsOld.value = structuredClone(lastCharChatLinesIxs.value);
 799        }
 800      }, [generating.value]);
 801
 802      const chatLine = ([user, data], index) => {
 803        let message
 804        const isArrayMessage = Array.isArray(data);
 805        const text = isArrayMessage ?
 806            data.map(msg => msg.content).join('') :
 807            data;
 808        if (params.value.n_probs > 0 && isArrayMessage) {
 809          message = html`<${Probabilities} data=${data} />`
 810        } else {
 811          message = isCompletionMode ?
 812            text :
 813            html`<${Markdownish} text=${template(text)} />`
 814        }
 815
 816        const fromBot = user && user === '{{char}}';
 817        if (fromBot && !lastCharChatLinesIxs.value.includes(index))
 818          lastCharChatLinesIxs.value.push(index);
 819
 820        if (user) {
 821          return html`
 822          <div>
 823            <p key=${index}><strong>${template(user)}:</strong> ${message}</p>
 824            ${
 825              fromBot && ttsVoice.value
 826              && html`<button disabled=${generating.value} onclick=${e => ttsChatLine(e, index, text)} aria-label=${ttsChatLineActiveIx.value === index ? 'Pause' : 'Play'}>${ ttsChatLineActiveIx.value === index ? '⏸️' : '▶️' }</div>`
 827            }
 828          </div>
 829          `;
 830        } else {
 831          return isCompletionMode ?
 832            html`<span key=${index}>${message}</span>` :
 833            html`<div><p key=${index}>${message}</p></div>`
 834        }
 835      };
 836
 837      const handleCompletionEdit = (e) => {
 838        session.value.prompt = e.target.innerText;
 839        session.value.transcript = [];
 840      }
 841
 842      return html`
 843        <div id="chat" ref=${container} key=${messages.length}>
 844          <img style="width: 60%;${!session.value.image_selected ? `display: none;` : ``}" src="${session.value.image_selected}"/>
 845          <span contenteditable=${isCompletionMode} ref=${container} oninput=${handleCompletionEdit}>
 846            ${messages.flatMap(chatLine)}
 847          </span>
 848        </div>`;
 849    };
 850
 851    const ConfigForm = (props) => {
 852      const updateSession = (el) => session.value = { ...session.value, [el.target.name]: el.target.value }
 853      const updateParams = (el) => params.value = { ...params.value, [el.target.name]: el.target.value }
 854      const updateParamsFloat = (el) => params.value = { ...params.value, [el.target.name]: parseFloat(el.target.value) }
 855      const updateParamsInt = (el) => params.value = { ...params.value, [el.target.name]: Math.floor(parseFloat(el.target.value)) }
 856      const updateParamsBool = (el) => params.value = { ...params.value, [el.target.name]: el.target.checked }
 857
 858      const grammarJsonSchemaPropOrder = signal('')
 859      const updateGrammarJsonSchemaPropOrder = (el) => grammarJsonSchemaPropOrder.value = el.target.value
 860      const convertJSONSchemaGrammar = async () => {
 861        try {
 862          let schema = JSON.parse(params.value.grammar)
 863          const converter = new SchemaConverter({
 864            prop_order: grammarJsonSchemaPropOrder.value
 865              .split(',')
 866              .reduce((acc, cur, i) => ({ ...acc, [cur.trim()]: i }), {}),
 867            allow_fetch: true,
 868          })
 869          schema = await converter.resolveRefs(schema, 'input')
 870          converter.visit(schema, '')
 871          params.value = {
 872            ...params.value,
 873            grammar: converter.formatGrammar(),
 874          }
 875        } catch (e) {
 876          alert(`Convert failed: ${e.message}`)
 877        }
 878      }
 879
 880      const FloatField = ({ label, max, min, name, step, value }) => {
 881        return html`
 882          <div>
 883            <label for="${name}">${label}</label>
 884            <input type="range" id="${name}" min="${min}" max="${max}" step="${step}" name="${name}" value="${value}" oninput=${updateParamsFloat} />
 885            <span>${value}</span>
 886          </div>
 887        `
 888      };
 889
 890      const IntField = ({ label, max, min, name, value }) => {
 891        return html`
 892          <div>
 893            <label for="${name}">${label}</label>
 894            <input type="range" id="${name}" min="${min}" max="${max}" name="${name}" value="${value}" oninput=${updateParamsInt} />
 895            <span>${value}</span>
 896          </div>
 897        `
 898      };
 899
 900      const BoolField = ({ label, name, value }) => {
 901        return html`
 902          <div>
 903            <label for="${name}">${label}</label>
 904            <input type="checkbox" id="${name}" name="${name}" checked="${value}" onclick=${updateParamsBool} />
 905          </div>
 906        `
 907      };
 908
 909      const userTemplateReset = (e) => {
 910        e.preventDefault();
 911        userTemplateResetToDefaultAndApply()
 912      }
 913
 914      const UserTemplateResetButton = () => {
 915        if (selectedUserTemplate.value.name == 'default') {
 916          return html`
 917            <button disabled>Using default template</button>
 918          `
 919        }
 920
 921        return html`
 922          <button onclick=${userTemplateReset}>Reset all to default</button>
 923        `
 924      };
 925
 926      useEffect(() => {
 927        // autosave template on every change
 928        userTemplateAutosave()
 929      }, [session.value, params.value])
 930
 931      const GrammarControl = () => (
 932        html`
 933          <div>
 934            <label for="template">Grammar</label>
 935            <textarea id="grammar" name="grammar" placeholder="Use gbnf or JSON Schema+convert" value="${params.value.grammar}" rows=4 oninput=${updateParams}/>
 936            <input type="text" name="prop-order" placeholder="order: prop1,prop2,prop3" oninput=${updateGrammarJsonSchemaPropOrder} />
 937            <button type="button" onclick=${convertJSONSchemaGrammar}>Convert JSON Schema</button>
 938          </div>
 939          `
 940      );
 941
 942      const PromptControlFieldSet = () => (
 943        html`
 944        <fieldset>
 945          <div>
 946            <label htmlFor="prompt">Prompt</label>
 947            <textarea type="text" name="prompt" value="${session.value.prompt}" oninput=${updateSession}/>
 948          </div>
 949        </fieldset>
 950        `
 951      );
 952
 953      const ChatConfigForm = () => (
 954        html`
 955          ${PromptControlFieldSet()}
 956
 957          <fieldset class="two">
 958            <div>
 959              <label for="user">User name</label>
 960              <input type="text" name="user" value="${session.value.user}" oninput=${updateSession} />
 961            </div>
 962
 963            <div>
 964              <label for="bot">Bot name</label>
 965              <input type="text" name="char" value="${session.value.char}" oninput=${updateSession} />
 966            </div>
 967          </fieldset>
 968
 969          <fieldset>
 970            <div>
 971              <label for="template">Prompt template</label>
 972              <textarea id="template" name="template" value="${session.value.template}" rows=4 oninput=${updateSession}/>
 973            </div>
 974
 975            <div>
 976              <label for="template">Chat history template</label>
 977              <textarea id="template" name="historyTemplate" value="${session.value.historyTemplate}" rows=1 oninput=${updateSession}/>
 978            </div>
 979            ${GrammarControl()}
 980          </fieldset>
 981      `
 982      );
 983
 984      const CompletionConfigForm = () => (
 985        html`
 986          ${PromptControlFieldSet()}
 987          <fieldset>${GrammarControl()}</fieldset>
 988        `
 989      );
 990
 991      return html`
 992        <form>
 993          <fieldset class="two">
 994            <${UserTemplateResetButton}/>
 995            <div>
 996              <label class="slim"><input type="radio" name="type" value="chat" checked=${session.value.type === "chat"} oninput=${updateSession} /> Chat</label>
 997              <label class="slim"><input type="radio" name="type" value="completion" checked=${session.value.type === "completion"} oninput=${updateSession} /> Completion</label>
 998            </div>
 999          </fieldset>
1000
1001          ${session.value.type === 'chat' ? ChatConfigForm() : CompletionConfigForm()}
1002
1003          <fieldset class="two">
1004            ${IntField({ label: "Predictions", max: 2048, min: -1, name: "n_predict", value: params.value.n_predict })}
1005            ${FloatField({ label: "Temperature", max: 2.0, min: 0.0, name: "temperature", step: 0.01, value: params.value.temperature })}
1006            ${FloatField({ label: "Penalize repeat sequence", max: 2.0, min: 0.0, name: "repeat_penalty", step: 0.01, value: params.value.repeat_penalty })}
1007            ${IntField({ label: "Consider N tokens for penalize", max: 2048, min: 0, name: "repeat_last_n", value: params.value.repeat_last_n })}
1008            ${IntField({ label: "Top-K sampling", max: 100, min: -1, name: "top_k", value: params.value.top_k })}
1009            ${FloatField({ label: "Top-P sampling", max: 1.0, min: 0.0, name: "top_p", step: 0.01, value: params.value.top_p })}
1010            ${FloatField({ label: "Min-P sampling", max: 1.0, min: 0.0, name: "min_p", step: 0.01, value: params.value.min_p })}
1011          </fieldset>
1012          <details>
1013            <summary>More options</summary>
1014            <fieldset class="two">
1015              ${FloatField({ label: "Typical P", max: 1.0, min: 0.0, name: "typical_p", step: 0.01, value: params.value.typical_p })}
1016              ${FloatField({ label: "Presence penalty", max: 1.0, min: 0.0, name: "presence_penalty", step: 0.01, value: params.value.presence_penalty })}
1017              ${FloatField({ label: "Frequency penalty", max: 1.0, min: 0.0, name: "frequency_penalty", step: 0.01, value: params.value.frequency_penalty })}
1018              ${FloatField({ label: "DRY Penalty Multiplier", max: 5.0, min: 0.0, name: "dry_multiplier", step: 0.01, value: params.value.dry_multiplier })}
1019              ${FloatField({ label: "DRY Base", max: 3.0, min: 1.0, name: "dry_base", step: 0.01, value: params.value.dry_base })}
1020              ${IntField({ label: "DRY Allowed Length", max: 10, min: 2, step: 1, name: "dry_allowed_length", value: params.value.dry_allowed_length })}
1021              ${IntField({ label: "DRY Penalty Last N", max: 2048, min: -1, step: 16, name: "dry_penalty_last_n", value: params.value.dry_penalty_last_n })}
1022              ${FloatField({ label: "XTC probability", max: 1.0, min: 0.0, name: "xtc_probability", step: 0.01, value: params.value.xtc_probability })}
1023              ${FloatField({ label: "XTC threshold", max: 0.5, min: 0.0, name: "xtc_threshold", step: 0.01, value: params.value.xtc_threshold })}
1024            </fieldset>
1025            <hr />
1026            <fieldset class="three">
1027              <div>
1028                <label><input type="radio" name="mirostat" value="0" checked=${params.value.mirostat == 0} oninput=${updateParamsInt} /> no Mirostat</label>
1029                <label><input type="radio" name="mirostat" value="1" checked=${params.value.mirostat == 1} oninput=${updateParamsInt} /> Mirostat v1</label>
1030                <label><input type="radio" name="mirostat" value="2" checked=${params.value.mirostat == 2} oninput=${updateParamsInt} /> Mirostat v2</label>
1031              </div>
1032              ${FloatField({ label: "Mirostat tau", max: 10.0, min: 0.0, name: "mirostat_tau", step: 0.01, value: params.value.mirostat_tau })}
1033              ${FloatField({ label: "Mirostat eta", max: 1.0, min: 0.0, name: "mirostat_eta", step: 0.01, value: params.value.mirostat_eta })}
1034            </fieldset>
1035            <fieldset>
1036              ${IntField({ label: "Show Probabilities", max: 10, min: 0, name: "n_probs", value: params.value.n_probs })}
1037            </fieldset>
1038            <fieldset>
1039              ${IntField({ label: "Min Probabilities from each Sampler", max: 10, min: 0, name: "min_keep", value: params.value.min_keep })}
1040            </fieldset>
1041            <fieldset>
1042              <label for="api_key">API Key</label>
1043              <input type="text" name="api_key" value="${params.value.api_key}" placeholder="Enter API key" oninput=${updateParams} />
1044            </fieldset>
1045          </details>
1046        </form>
1047      `
1048    }
1049
1050    const probColor = (p) => {
1051      const r = Math.floor(192 * (1 - p));
1052      const g = Math.floor(192 * p);
1053      return `rgba(${r},${g},0,0.3)`;
1054    }
1055
1056    const Probabilities = (params) => {
1057      return params.data.map(msg => {
1058        const { completion_probabilities } = msg;
1059        if (
1060          !completion_probabilities ||
1061          completion_probabilities.length === 0
1062        ) return msg.content
1063
1064        if (completion_probabilities.length > 1) {
1065          // Not for byte pair
1066          if (completion_probabilities[0].content.startsWith('byte: \\')) return msg.content
1067
1068          const splitData = completion_probabilities.map(prob => ({
1069            content: prob.content,
1070            completion_probabilities: [prob]
1071          }))
1072          return html`<${Probabilities} data=${splitData} />`
1073        }
1074
1075        const { probs, content } = completion_probabilities[0]
1076        const found = probs.find(p => p.tok_str === msg.content)
1077        const pColor = found ? probColor(found.prob) : 'transparent'
1078
1079        const popoverChildren = html`
1080          <div class="prob-set">
1081            ${probs.map((p, index) => {
1082          return html`
1083                <div
1084                  key=${index}
1085                  title=${`prob: ${p.prob}`}
1086                  style=${{
1087              padding: '0.3em',
1088              backgroundColor: p.tok_str === content ? probColor(p.prob) : 'transparent'
1089            }}
1090                >
1091                  <span>${p.tok_str}: </span>
1092                  <span>${Math.floor(p.prob * 100)}%</span>
1093                </div>
1094              `
1095        })}
1096          </div>
1097        `
1098
1099        return html`
1100          <${Popover} style=${{ backgroundColor: pColor }} popoverChildren=${popoverChildren}>
1101            ${msg.content.match(/\n/gim) ? html`<br />` : msg.content}
1102          </>
1103        `
1104      });
1105    }
1106
1107    // poor mans markdown replacement
1108    const Markdownish = (params) => {
1109      const chunks = params.text.split('```');
1110
1111      for (let i = 0; i < chunks.length; i++) {
1112        if (i % 2 === 0) { // outside code block
1113          chunks[i] = chunks[i]
1114          .replace(/&/g, '&amp;')
1115          .replace(/</g, '&lt;')
1116          .replace(/>/g, '&gt;')
1117          .replace(/(^|\n)#{1,6} ([^\n]*)(?=([^`]*`[^`]*`)*[^`]*$)/g, '$1<h3>$2</h3>')
1118          .replace(/\*\*(.*?)\*\*(?=([^`]*`[^`]*`)*[^`]*$)/g, '<strong>$1</strong>')
1119          .replace(/__(.*?)__(?=([^`]*`[^`]*`)*[^`]*$)/g, '<strong>$1</strong>')
1120          .replace(/\*(.*?)\*(?=([^`]*`[^`]*`)*[^`]*$)/g, '<em>$1</em>')
1121          .replace(/_(.*?)_(?=([^`]*`[^`]*`)*[^`]*$)/g, '<em>$1</em>')
1122          .replace(/```.*?\n([\s\S]*?)```/g, '<pre><code>$1</code></pre>')
1123          .replace(/`(.*?)`/g, '<code>$1</code>')
1124          .replace(/\n/gim, '<br />');
1125        } else { // inside code block
1126          chunks[i] = `<pre><code>${chunks[i]}</code></pre>`;
1127        }
1128      }
1129
1130      const restoredText = chunks.join('');
1131
1132      return html`<span dangerouslySetInnerHTML=${{ __html: restoredText }} />`;
1133    };
1134
1135    const ModelGenerationInfo = (params) => {
1136      if (!llamaStats.value) {
1137        return html`<span/>`
1138      }
1139      return html`
1140        <span>
1141          ${llamaStats.value.tokens_predicted} predicted, ${llamaStats.value.tokens_cached} cached, ${llamaStats.value.timings.predicted_per_token_ms.toFixed()}ms per token, ${llamaStats.value.timings.predicted_per_second.toFixed(2)} tokens per second
1142        </span>
1143      `
1144    }
1145
1146
1147    // simple popover impl
1148    const Popover = (props) => {
1149      const isOpen = useSignal(false);
1150      const position = useSignal({ top: '0px', left: '0px' });
1151      const buttonRef = useRef(null);
1152      const popoverRef = useRef(null);
1153
1154      const togglePopover = () => {
1155        if (buttonRef.current) {
1156          const rect = buttonRef.current.getBoundingClientRect();
1157          position.value = {
1158            top: `${rect.bottom + window.scrollY}px`,
1159            left: `${rect.left + window.scrollX}px`,
1160          };
1161        }
1162        isOpen.value = !isOpen.value;
1163      };
1164
1165      const handleClickOutside = (event) => {
1166        if (popoverRef.current && !popoverRef.current.contains(event.target) && !buttonRef.current.contains(event.target)) {
1167          isOpen.value = false;
1168        }
1169      };
1170
1171      useEffect(() => {
1172        document.addEventListener('mousedown', handleClickOutside);
1173        return () => {
1174          document.removeEventListener('mousedown', handleClickOutside);
1175        };
1176      }, []);
1177
1178      return html`
1179        <span style=${props.style} ref=${buttonRef} onClick=${togglePopover}>${props.children}</span>
1180        ${isOpen.value && html`
1181          <${Portal} into="#portal">
1182            <div
1183              ref=${popoverRef}
1184              class="popover-content"
1185              style=${{
1186            top: position.value.top,
1187            left: position.value.left,
1188          }}
1189            >
1190              ${props.popoverChildren}
1191            </div>
1192          </${Portal}>
1193        `}
1194      `;
1195    };
1196
1197    // Source: preact-portal (https://github.com/developit/preact-portal/blob/master/src/preact-portal.js)
1198    /** Redirect rendering of descendants into the given CSS selector */
1199    class Portal extends Component {
1200      componentDidUpdate(props) {
1201        for (let i in props) {
1202          if (props[i] !== this.props[i]) {
1203            return setTimeout(this.renderLayer);
1204          }
1205        }
1206      }
1207
1208      componentDidMount() {
1209        this.isMounted = true;
1210        this.renderLayer = this.renderLayer.bind(this);
1211        this.renderLayer();
1212      }
1213
1214      componentWillUnmount() {
1215        this.renderLayer(false);
1216        this.isMounted = false;
1217        if (this.remote && this.remote.parentNode) this.remote.parentNode.removeChild(this.remote);
1218      }
1219
1220      findNode(node) {
1221        return typeof node === 'string' ? document.querySelector(node) : node;
1222      }
1223
1224      renderLayer(show = true) {
1225        if (!this.isMounted) return;
1226
1227        // clean up old node if moving bases:
1228        if (this.props.into !== this.intoPointer) {
1229          this.intoPointer = this.props.into;
1230          if (this.into && this.remote) {
1231            this.remote = render(html`<${PortalProxy} />`, this.into, this.remote);
1232          }
1233          this.into = this.findNode(this.props.into);
1234        }
1235
1236        this.remote = render(html`
1237          <${PortalProxy} context=${this.context}>
1238            ${show && this.props.children || null}
1239          </${PortalProxy}>
1240        `, this.into, this.remote);
1241      }
1242
1243      render() {
1244        return null;
1245      }
1246    }
1247    // high-order component that renders its first child if it exists.
1248    // used as a conditional rendering proxy.
1249    class PortalProxy extends Component {
1250      getChildContext() {
1251        return this.props.context;
1252      }
1253      render({ children }) {
1254        return children || null;
1255      }
1256    }
1257
1258    function App(props) {
1259      useEffect(() => {
1260        const query = new URLSearchParams(location.search).get("q");
1261        if (query) chat(query);
1262      }, []);
1263
1264      return html`
1265        <div class="mode-${session.value.type}">
1266          <header>
1267            <div class="grid-container">
1268              <div class="grid-item"></div>
1269              <div class="grid-item"><h1>llama.cpp</h1></div>
1270              <div class="grid-item"><a class="customlink" href="index-new.html">New UI</a></div>
1271            </div>
1272          </header>
1273
1274          <main id="content">
1275            <${chatStarted.value ? ChatLog : ConfigForm} />
1276          </main>
1277
1278          <section id="write">
1279            <${session.value.type === 'chat' ? MessageInput : CompletionControls} />
1280          </section>
1281
1282          <footer>
1283            <p><${ModelGenerationInfo} /></p>
1284            <p>Powered by <a href="https://github.com/ggml-org/llama.cpp">llama.cpp</a> and <a href="https://ggml.ai">ggml.ai</a>.</p>
1285          </footer>
1286        </div>
1287      `;
1288    }
1289
1290    render(h(App), document.querySelector('#container'));
1291  </script>
1292</head>
1293
1294<body>
1295  <div id="container">
1296    <input type="file" id="fileInput" accept="image/*" style="display: none;">
1297  </div>
1298  <div id="portal"></div>
1299</body>
1300
1301</html>