llmnpc - llama.cpp/tools/server/themes/buttons-top/index.html

   1<html>
   2
   3<head>
   4  <meta charset="UTF-8">
   5  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1" />
   6  <meta name="color-scheme" content="light dark">
   7  <title>llama.cpp - chat</title>
   8
   9  <style>
  10    body {
  11      font-family: system-ui;
  12      font-size: 90%;
  13    }
  14
  15    #container {
  16      margin: 0em auto;
  17      display: flex;
  18      flex-direction: column;
  19      justify-content: space-between;
  20      height: 100%;
  21    }
  22
  23    main {
  24      margin: 3px;
  25      display: flex;
  26      flex-direction: column;
  27      justify-content: space-between;
  28      gap: 1em;
  29
  30      flex-grow: 1;
  31      overflow-y: auto;
  32
  33      border: 1px solid #ccc;
  34      border-radius: 5px;
  35      padding: 0.5em;
  36    }
  37
  38    body {
  39      max-width: 600px;
  40      min-width: 300px;
  41      line-height: 1.2;
  42      margin: 0 auto;
  43      padding: 0 0.5em;
  44    }
  45
  46    p {
  47      overflow-wrap: break-word;
  48      word-wrap: break-word;
  49      hyphens: auto;
  50      margin-top: 0.5em;
  51      margin-bottom: 0.5em;
  52    }
  53
  54    #write form {
  55      margin: 1em 0 0 0;
  56      display: flex;
  57      flex-direction: column;
  58      gap: 0.5em;
  59      align-items: stretch;
  60    }
  61
  62    .right {
  63      display: flex;
  64      flex-direction: row;
  65      gap: 0.5em;
  66      justify-content: flex-end;
  67    }
  68
  69    fieldset {
  70      border: none;
  71      padding: 0;
  72      margin: 0;
  73    }
  74
  75    fieldset.two {
  76      display: grid;
  77      grid-template: "a a";
  78      gap: 1em;
  79    }
  80
  81    fieldset.three {
  82      display: grid;
  83      grid-template: "a a a";
  84      gap: 1em;
  85    }
  86
  87    details {
  88      border: 1px solid #aaa;
  89      border-radius: 4px;
  90      padding: 0.5em 0.5em 0;
  91      margin-top: 0.5em;
  92    }
  93
  94    summary {
  95      font-weight: bold;
  96      margin: -0.5em -0.5em 0;
  97      padding: 0.5em;
  98      cursor: pointer;
  99    }
 100
 101    details[open] {
 102      padding: 0.5em;
 103    }
 104
 105    .prob-set {
 106      padding: 0.3em;
 107      border-bottom: 1px solid #ccc;
 108    }
 109
 110    .popover-content {
 111      position: absolute;
 112      background-color: white;
 113      padding: 0.2em;
 114      box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
 115    }
 116
 117    textarea {
 118      padding: 5px;
 119      flex-grow: 1;
 120      width: 100%;
 121    }
 122
 123    pre code {
 124      display: block;
 125      background-color: #222;
 126      color: #ddd;
 127    }
 128
 129    code {
 130      font-family: monospace;
 131      padding: 0.1em 0.3em;
 132      border-radius: 3px;
 133    }
 134
 135    fieldset label {
 136      margin: 0.5em 0;
 137      display: block;
 138    }
 139
 140    fieldset label.slim {
 141      margin: 0 0.5em;
 142      display: inline;
 143    }
 144
 145    header,
 146    footer {
 147      text-align: center;
 148    }
 149
 150    footer {
 151      font-size: 80%;
 152      color: #888;
 153    }
 154
 155    .mode-chat textarea[name=prompt] {
 156      height: 4.5em;
 157    }
 158
 159    .mode-completion textarea[name=prompt] {
 160      height: 10em;
 161    }
 162
 163    [contenteditable] {
 164      display: inline-block;
 165      white-space: pre-wrap;
 166      outline: 0px solid transparent;
 167    }
 168
 169    @keyframes loading-bg-wipe {
 170      0% {
 171        background-position: 0%;
 172      }
 173
 174      100% {
 175        background-position: 100%;
 176      }
 177    }
 178
 179    .loading {
 180      --loading-color-1: #eeeeee00;
 181      --loading-color-2: #eeeeeeff;
 182      background-size: 50% 100%;
 183      background-image: linear-gradient(90deg, var(--loading-color-1), var(--loading-color-2), var(--loading-color-1));
 184      animation: loading-bg-wipe 2s linear infinite;
 185    }
 186
 187    @media (prefers-color-scheme: dark) {
 188      .loading {
 189        --loading-color-1: #22222200;
 190        --loading-color-2: #222222ff;
 191      }
 192
 193      .popover-content {
 194        background-color: black;
 195      }
 196    }
 197  </style>
 198
 199  <script type="module">
 200    import {
 201      html, h, signal, effect, computed, render, useSignal, useEffect, useRef, Component
 202    } from './index.js';
 203
 204    import { llama } from './completion.js';
 205    import { SchemaConverter } from './json-schema-to-grammar.mjs';
 206    let selected_image = false;
 207    var slot_id = -1;
 208
 209    const session = signal({
 210      prompt: "This is a conversation between User and Llama, a friendly chatbot. Llama is helpful, kind, honest, good at writing, and never fails to answer any requests immediately and with precision.",
 211      template: "{{prompt}}\n\n{{history}}\n{{char}}:",
 212      historyTemplate: "{{name}}: {{message}}",
 213      transcript: [],
 214      type: "chat",  // "chat" | "completion"
 215      char: "Llama",
 216      user: "User",
 217      image_selected: ''
 218    })
 219
 220    const params = signal({
 221      n_predict: 400,
 222      temperature: 0.7,
 223      repeat_last_n: 256, // 0 = disable penalty, -1 = context size
 224      repeat_penalty: 1.18, // 1.0 = disabled
 225      top_k: 40, // <= 0 to use vocab size
 226      top_p: 0.95, // 1.0 = disabled
 227      min_p: 0.05, // 0 = disabled
 228      typical_p: 1.0, // 1.0 = disabled
 229      presence_penalty: 0.0, // 0.0 = disabled
 230      frequency_penalty: 0.0, // 0.0 = disabled
 231      mirostat: 0, // 0/1/2
 232      mirostat_tau: 5, // target entropy
 233      mirostat_eta: 0.1, // learning rate
 234      grammar: '',
 235      n_probs: 0, // no completion_probabilities,
 236      min_keep: 0, // min probs from each sampler,
 237      image_data: [],
 238      cache_prompt: true,
 239      api_key: ''
 240    })
 241
 242    /* START: Support for storing prompt templates and parameters in browsers LocalStorage */
 243
 244    const local_storage_storageKey = "llamacpp_server_local_storage";
 245
 246    function local_storage_setDataFromObject(tag, content) {
 247      localStorage.setItem(local_storage_storageKey + '/' + tag, JSON.stringify(content));
 248    }
 249
 250    function local_storage_setDataFromRawText(tag, content) {
 251      localStorage.setItem(local_storage_storageKey + '/' + tag, content);
 252    }
 253
 254    function local_storage_getDataAsObject(tag) {
 255      const item = localStorage.getItem(local_storage_storageKey + '/' + tag);
 256      if (!item) {
 257        return null;
 258      } else {
 259        return JSON.parse(item);
 260      }
 261    }
 262
 263    function local_storage_getDataAsRawText(tag) {
 264      const item = localStorage.getItem(local_storage_storageKey + '/' + tag);
 265      if (!item) {
 266        return null;
 267      } else {
 268        return item;
 269      }
 270    }
 271
 272    // create a container for user templates and settings
 273
 274    const savedUserTemplates = signal({})
 275    const selectedUserTemplate = signal({ name: '', template: { session: {}, params: {} } })
 276
 277    // let's import locally saved templates and settings if there are any
 278    // user templates and settings are stored in one object
 279    // in form of { "templatename": "templatedata" } and { "settingstemplatename":"settingsdata" }
 280
 281    console.log('Importing saved templates')
 282
 283    let importedTemplates = local_storage_getDataAsObject('user_templates')
 284
 285    if (importedTemplates) {
 286      // saved templates were successfully imported.
 287
 288      console.log('Processing saved templates and updating default template')
 289      params.value = { ...params.value, image_data: [] };
 290
 291      //console.log(importedTemplates);
 292      savedUserTemplates.value = importedTemplates;
 293
 294      //override default template
 295      savedUserTemplates.value.default = { session: session.value, params: params.value }
 296      local_storage_setDataFromObject('user_templates', savedUserTemplates.value)
 297    } else {
 298      // no saved templates detected.
 299
 300      console.log('Initializing LocalStorage and saving default template')
 301
 302      savedUserTemplates.value = { "default": { session: session.value, params: params.value } }
 303      local_storage_setDataFromObject('user_templates', savedUserTemplates.value)
 304    }
 305
 306    function userTemplateResetToDefault() {
 307      console.log('Resetting template to default')
 308      selectedUserTemplate.value.name = 'default';
 309      selectedUserTemplate.value.data = savedUserTemplates.value['default'];
 310    }
 311
 312    function userTemplateApply(t) {
 313      session.value = t.data.session;
 314      session.value = { ...session.value, image_selected: '' };
 315      params.value = t.data.params;
 316      params.value = { ...params.value, image_data: [] };
 317    }
 318
 319    function userTemplateResetToDefaultAndApply() {
 320      userTemplateResetToDefault()
 321      userTemplateApply(selectedUserTemplate.value)
 322    }
 323
 324    function userTemplateLoadAndApplyAutosaved() {
 325      // get autosaved last used template
 326      let lastUsedTemplate = local_storage_getDataAsObject('user_templates_last')
 327
 328      if (lastUsedTemplate) {
 329
 330        console.log('Autosaved template found, restoring')
 331
 332        selectedUserTemplate.value = lastUsedTemplate
 333      }
 334      else {
 335
 336        console.log('No autosaved template found, using default template')
 337        // no autosaved last used template was found, so load from default.
 338
 339        userTemplateResetToDefault()
 340      }
 341
 342      console.log('Applying template')
 343      // and update internal data from templates
 344
 345      userTemplateApply(selectedUserTemplate.value)
 346    }
 347
 348    //console.log(savedUserTemplates.value)
 349    //console.log(selectedUserTemplate.value)
 350
 351    function userTemplateAutosave() {
 352      console.log('Template Autosave...')
 353      if (selectedUserTemplate.value.name == 'default') {
 354        // we don't want to save over default template, so let's create a new one
 355        let newTemplateName = 'UserTemplate-' + Date.now().toString()
 356        let newTemplate = { 'name': newTemplateName, 'data': { 'session': session.value, 'params': params.value } }
 357
 358        console.log('Saving as ' + newTemplateName)
 359
 360        // save in the autosave slot
 361        local_storage_setDataFromObject('user_templates_last', newTemplate)
 362
 363        // and load it back and apply
 364        userTemplateLoadAndApplyAutosaved()
 365      } else {
 366        local_storage_setDataFromObject('user_templates_last', { 'name': selectedUserTemplate.value.name, 'data': { 'session': session.value, 'params': params.value } })
 367      }
 368    }
 369
 370    console.log('Checking for autosaved last used template')
 371    userTemplateLoadAndApplyAutosaved()
 372
 373    /* END: Support for storing prompt templates and parameters in browsers LocalStorage */
 374
 375    const llamaStats = signal(null)
 376    const controller = signal(null)
 377
 378    // currently generating a completion?
 379    const generating = computed(() => controller.value != null)
 380
 381    // has the user started a chat?
 382    const chatStarted = computed(() => session.value.transcript.length > 0)
 383
 384    const transcriptUpdate = (transcript) => {
 385      session.value = {
 386        ...session.value,
 387        transcript
 388      }
 389    }
 390
 391    // simple template replace
 392    const template = (str, extraSettings) => {
 393      let settings = session.value;
 394      if (extraSettings) {
 395        settings = { ...settings, ...extraSettings };
 396      }
 397      return String(str).replaceAll(/\{\{(.*?)\}\}/g, (_, key) => template(settings[key]));
 398    }
 399
 400    async function runLlama(prompt, llamaParams, char) {
 401      const currentMessages = [];
 402      const history = session.value.transcript;
 403      if (controller.value) {
 404        throw new Error("already running");
 405      }
 406      controller.value = new AbortController();
 407      for await (const chunk of llama(prompt, llamaParams, { controller: controller.value, api_url: location.pathname.replace(/\/+$/, '') })) {
 408        const data = chunk.data;
 409
 410        if (data.stop) {
 411          while (
 412            currentMessages.length > 0 &&
 413            currentMessages[currentMessages.length - 1].content.match(/\n$/) != null
 414          ) {
 415            currentMessages.pop();
 416          }
 417          transcriptUpdate([...history, [char, currentMessages]])
 418          console.log("Completion finished: '", currentMessages.map(msg => msg.content).join(''), "', summary: ", data);
 419        } else {
 420          currentMessages.push(data);
 421          slot_id = data.slot_id;
 422          if (selected_image && !data.multimodal) {
 423            alert("The server was not compiled for multimodal or the model projector can't be loaded.");
 424            return;
 425          }
 426          transcriptUpdate([...history, [char, currentMessages]])
 427        }
 428
 429        if (data.timings) {
 430          llamaStats.value = data;
 431        }
 432      }
 433
 434      controller.value = null;
 435    }
 436
 437    // send message to server
 438    const chat = async (msg) => {
 439      if (controller.value) {
 440        console.log('already running...');
 441        return;
 442      }
 443
 444      transcriptUpdate([...session.value.transcript, ["{{user}}", msg]])
 445
 446      let prompt = template(session.value.template, {
 447        message: msg,
 448        history: session.value.transcript.flatMap(
 449          ([name, data]) =>
 450            template(
 451              session.value.historyTemplate,
 452              {
 453                name,
 454                message: Array.isArray(data) ?
 455                  data.map(msg => msg.content).join('').replace(/^\s/, '') :
 456                  data,
 457              }
 458            )
 459        ).join("\n"),
 460      });
 461      if (selected_image) {
 462        prompt = `A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\nUSER:[img-10]${msg}\nASSISTANT:`;
 463      }
 464      await runLlama(prompt, {
 465        ...params.value,
 466        slot_id: slot_id,
 467        stop: ["</s>", template("{{char}}:"), template("{{user}}:")],
 468      }, "{{char}}");
 469    }
 470
 471    const runCompletion = () => {
 472      if (controller.value) {
 473        console.log('already running...');
 474        return;
 475      }
 476      const { prompt } = session.value;
 477      transcriptUpdate([...session.value.transcript, ["", prompt]]);
 478      runLlama(prompt, {
 479        ...params.value,
 480        slot_id: slot_id,
 481        stop: [],
 482      }, "").finally(() => {
 483        session.value.prompt = session.value.transcript.map(([_, data]) =>
 484          Array.isArray(data) ? data.map(msg => msg.content).join('') : data
 485        ).join('');
 486        session.value.transcript = [];
 487      })
 488    }
 489
 490    const stop = (e) => {
 491      e.preventDefault();
 492      if (controller.value) {
 493        controller.value.abort();
 494        controller.value = null;
 495      }
 496    }
 497
 498    const reset = (e) => {
 499      stop(e);
 500      transcriptUpdate([]);
 501    }
 502
 503    const uploadImage = (e) => {
 504      e.preventDefault();
 505      document.getElementById("fileInput").click();
 506      document.getElementById("fileInput").addEventListener("change", function (event) {
 507        const selectedFile = event.target.files[0];
 508        if (selectedFile) {
 509          const reader = new FileReader();
 510          reader.onload = function () {
 511            const image_data = reader.result;
 512            session.value = { ...session.value, image_selected: image_data };
 513            params.value = {
 514              ...params.value, image_data: [
 515                { data: image_data.replace(/data:image\/[^;]+;base64,/, ''), id: 10 }]
 516            }
 517          };
 518          selected_image = true;
 519          reader.readAsDataURL(selectedFile);
 520        }
 521      });
 522    }
 523
 524    function MessageInput() {
 525      const message = useSignal("")
 526
 527      const submit = (e) => {
 528        stop(e);
 529        chat(message.value);
 530        message.value = "";
 531      }
 532
 533      const enterSubmits = (event) => {
 534        if (event.which === 13 && !event.shiftKey) {
 535          submit(event);
 536        }
 537      }
 538
 539      return html`
 540        <form onsubmit=${submit}>
 541          <div>
 542            <textarea
 543               className=${generating.value ? "loading" : null}
 544               oninput=${(e) => message.value = e.target.value}
 545               onkeypress=${enterSubmits}
 546               placeholder="Say something..."
 547               rows=2
 548               type="text"
 549               value="${message}"
 550            />
 551          </div>
 552          <div class="right">
 553            <button type="submit" disabled=${generating.value}>Send</button>
 554            <button onclick=${uploadImage}>Upload Image</button>
 555            <button onclick=${stop} disabled=${!generating.value}>Stop</button>
 556            <button onclick=${reset}>Reset</button>
 557          </div>
 558        </form>
 559      `
 560    }
 561
 562    function CompletionControls() {
 563      const submit = (e) => {
 564        stop(e);
 565        runCompletion();
 566      }
 567      return html`
 568        <div>
 569          <button onclick=${submit} type="button" disabled=${generating.value}>Start</button>
 570          <button onclick=${stop} disabled=${!generating.value}>Stop</button>
 571          <button onclick=${reset}>Reset</button>
 572        </div>`;
 573    }
 574
 575    const ChatLog = (props) => {
 576      const messages = session.value.transcript;
 577      const container = useRef(null)
 578
 579      useEffect(() => {
 580        // scroll to bottom (if needed)
 581        const parent = container.current.parentElement;
 582        if (parent && parent.scrollHeight <= parent.scrollTop + parent.offsetHeight + 300) {
 583          parent.scrollTo(0, parent.scrollHeight)
 584        }
 585      }, [messages])
 586
 587      const isCompletionMode = session.value.type === 'completion'
 588      const chatLine = ([user, data], index) => {
 589        let message
 590        const isArrayMessage = Array.isArray(data)
 591        if (params.value.n_probs > 0 && isArrayMessage) {
 592          message = html`<${Probabilities} data=${data} />`
 593        } else {
 594          const text = isArrayMessage ?
 595            data.map(msg => msg.content).join('').replace(/^\s+/, '') :
 596            data;
 597          message = isCompletionMode ?
 598            text :
 599            html`<${Markdownish} text=${template(text)} />`
 600        }
 601        if (user) {
 602          return html`<p key=${index}><strong>${template(user)}:</strong> ${message}</p>`
 603        } else {
 604          return isCompletionMode ?
 605            html`<span key=${index}>${message}</span>` :
 606            html`<p key=${index}>${message}</p>`
 607        }
 608      };
 609
 610      const handleCompletionEdit = (e) => {
 611        session.value.prompt = e.target.innerText;
 612        session.value.transcript = [];
 613      }
 614
 615      return html`
 616        <div id="chat" ref=${container} key=${messages.length}>
 617          <img style="width: 60%;${!session.value.image_selected ? `display: none;` : ``}" src="${session.value.image_selected}"/>
 618          <span contenteditable=${isCompletionMode} ref=${container} oninput=${handleCompletionEdit}>
 619            ${messages.flatMap(chatLine)}
 620          </span>
 621        </div>`;
 622    };
 623
 624    const ConfigForm = (props) => {
 625      const updateSession = (el) => session.value = { ...session.value, [el.target.name]: el.target.value }
 626      const updateParams = (el) => params.value = { ...params.value, [el.target.name]: el.target.value }
 627      const updateParamsFloat = (el) => params.value = { ...params.value, [el.target.name]: parseFloat(el.target.value) }
 628      const updateParamsInt = (el) => params.value = { ...params.value, [el.target.name]: Math.floor(parseFloat(el.target.value)) }
 629      const updateParamsBool = (el) => params.value = { ...params.value, [el.target.name]: el.target.checked }
 630
 631      const grammarJsonSchemaPropOrder = signal('')
 632      const updateGrammarJsonSchemaPropOrder = (el) => grammarJsonSchemaPropOrder.value = el.target.value
 633      const convertJSONSchemaGrammar = async () => {
 634        try {
 635          let schema = JSON.parse(params.value.grammar)
 636          const converter = new SchemaConverter({
 637            prop_order: grammarJsonSchemaPropOrder.value
 638              .split(',')
 639              .reduce((acc, cur, i) => ({ ...acc, [cur.trim()]: i }), {}),
 640            allow_fetch: true,
 641          })
 642          schema = await converter.resolveRefs(schema, 'input')
 643          converter.visit(schema, '')
 644          params.value = {
 645            ...params.value,
 646            grammar: converter.formatGrammar(),
 647          }
 648        } catch (e) {
 649          alert(`Convert failed: ${e.message}`)
 650        }
 651      }
 652
 653      const FloatField = ({ label, max, min, name, step, value }) => {
 654        return html`
 655          <div>
 656            <label for="${name}">${label}</label>
 657            <input type="range" id="${name}" min="${min}" max="${max}" step="${step}" name="${name}" value="${value}" oninput=${updateParamsFloat} />
 658            <span>${value}</span>
 659          </div>
 660        `
 661      };
 662
 663      const IntField = ({ label, max, min, name, value }) => {
 664        return html`
 665          <div>
 666            <label for="${name}">${label}</label>
 667            <input type="range" id="${name}" min="${min}" max="${max}" name="${name}" value="${value}" oninput=${updateParamsInt} />
 668            <span>${value}</span>
 669          </div>
 670        `
 671      };
 672
 673      const BoolField = ({ label, name, value }) => {
 674        return html`
 675          <div>
 676            <label for="${name}">${label}</label>
 677            <input type="checkbox" id="${name}" name="${name}" checked="${value}" onclick=${updateParamsBool} />
 678          </div>
 679        `
 680      };
 681
 682      const userTemplateReset = (e) => {
 683        e.preventDefault();
 684        userTemplateResetToDefaultAndApply()
 685      }
 686
 687      const UserTemplateResetButton = () => {
 688        if (selectedUserTemplate.value.name == 'default') {
 689          return html`
 690            <button disabled>Using default template</button>
 691          `
 692        }
 693
 694        return html`
 695          <button onclick=${userTemplateReset}>Reset all to default</button>
 696        `
 697      };
 698
 699      useEffect(() => {
 700        // autosave template on every change
 701        userTemplateAutosave()
 702      }, [session.value, params.value])
 703
 704      const GrammarControl = () => (
 705        html`
 706          <div>
 707            <label for="template">Grammar</label>
 708            <textarea id="grammar" name="grammar" placeholder="Use gbnf or JSON Schema+convert" value="${params.value.grammar}" rows=4 oninput=${updateParams}/>
 709            <input type="text" name="prop-order" placeholder="order: prop1,prop2,prop3" oninput=${updateGrammarJsonSchemaPropOrder} />
 710            <button type="button" onclick=${convertJSONSchemaGrammar}>Convert JSON Schema</button>
 711          </div>
 712          `
 713      );
 714
 715      const PromptControlFieldSet = () => (
 716        html`
 717        <fieldset>
 718          <div>
 719            <label htmlFor="prompt">Prompt</label>
 720            <textarea type="text" name="prompt" value="${session.value.prompt}" oninput=${updateSession}/>
 721          </div>
 722        </fieldset>
 723        `
 724      );
 725
 726      const ChatConfigForm = () => (
 727        html`
 728          ${PromptControlFieldSet()}
 729
 730          <fieldset class="two">
 731            <div>
 732              <label for="user">User name</label>
 733              <input type="text" name="user" value="${session.value.user}" oninput=${updateSession} />
 734            </div>
 735
 736            <div>
 737              <label for="bot">Bot name</label>
 738              <input type="text" name="char" value="${session.value.char}" oninput=${updateSession} />
 739            </div>
 740          </fieldset>
 741
 742          <fieldset>
 743            <div>
 744              <label for="template">Prompt template</label>
 745              <textarea id="template" name="template" value="${session.value.template}" rows=4 oninput=${updateSession}/>
 746            </div>
 747
 748            <div>
 749              <label for="template">Chat history template</label>
 750              <textarea id="template" name="historyTemplate" value="${session.value.historyTemplate}" rows=1 oninput=${updateSession}/>
 751            </div>
 752            ${GrammarControl()}
 753          </fieldset>
 754      `
 755      );
 756
 757      const CompletionConfigForm = () => (
 758        html`
 759          ${PromptControlFieldSet()}
 760          <fieldset>${GrammarControl()}</fieldset>
 761        `
 762      );
 763
 764      return html`
 765        <form>
 766          <fieldset class="two">
 767            <${UserTemplateResetButton}/>
 768            <div>
 769              <label class="slim"><input type="radio" name="type" value="chat" checked=${session.value.type === "chat"} oninput=${updateSession} /> Chat</label>
 770              <label class="slim"><input type="radio" name="type" value="completion" checked=${session.value.type === "completion"} oninput=${updateSession} /> Completion</label>
 771            </div>
 772          </fieldset>
 773
 774          ${session.value.type === 'chat' ? ChatConfigForm() : CompletionConfigForm()}
 775
 776          <fieldset class="two">
 777            ${IntField({ label: "Predictions", max: 2048, min: -1, name: "n_predict", value: params.value.n_predict })}
 778            ${FloatField({ label: "Temperature", max: 2.0, min: 0.0, name: "temperature", step: 0.01, value: params.value.temperature })}
 779            ${FloatField({ label: "Penalize repeat sequence", max: 2.0, min: 0.0, name: "repeat_penalty", step: 0.01, value: params.value.repeat_penalty })}
 780            ${IntField({ label: "Consider N tokens for penalize", max: 2048, min: 0, name: "repeat_last_n", value: params.value.repeat_last_n })}
 781            ${IntField({ label: "Top-K sampling", max: 100, min: -1, name: "top_k", value: params.value.top_k })}
 782            ${FloatField({ label: "Top-P sampling", max: 1.0, min: 0.0, name: "top_p", step: 0.01, value: params.value.top_p })}
 783            ${FloatField({ label: "Min-P sampling", max: 1.0, min: 0.0, name: "min_p", step: 0.01, value: params.value.min_p })}
 784          </fieldset>
 785          <details>
 786            <summary>More options</summary>
 787            <fieldset class="two">
 788              ${FloatField({ label: "Typical P", max: 1.0, min: 0.0, name: "typical_p", step: 0.01, value: params.value.typical_p })}
 789              ${FloatField({ label: "Presence penalty", max: 1.0, min: 0.0, name: "presence_penalty", step: 0.01, value: params.value.presence_penalty })}
 790              ${FloatField({ label: "Frequency penalty", max: 1.0, min: 0.0, name: "frequency_penalty", step: 0.01, value: params.value.frequency_penalty })}
 791            </fieldset>
 792            <hr />
 793            <fieldset class="three">
 794              <div>
 795                <label><input type="radio" name="mirostat" value="0" checked=${params.value.mirostat == 0} oninput=${updateParamsInt} /> no Mirostat</label>
 796                <label><input type="radio" name="mirostat" value="1" checked=${params.value.mirostat == 1} oninput=${updateParamsInt} /> Mirostat v1</label>
 797                <label><input type="radio" name="mirostat" value="2" checked=${params.value.mirostat == 2} oninput=${updateParamsInt} /> Mirostat v2</label>
 798              </div>
 799              ${FloatField({ label: "Mirostat tau", max: 10.0, min: 0.0, name: "mirostat_tau", step: 0.01, value: params.value.mirostat_tau })}
 800              ${FloatField({ label: "Mirostat eta", max: 1.0, min: 0.0, name: "mirostat_eta", step: 0.01, value: params.value.mirostat_eta })}
 801            </fieldset>
 802            <fieldset>
 803              ${IntField({ label: "Show Probabilities", max: 10, min: 0, name: "n_probs", value: params.value.n_probs })}
 804            </fieldset>
 805            <fieldset>
 806              ${IntField({ label: "Min Probabilities from each Sampler", max: 10, min: 0, name: "min_keep", value: params.value.min_keep })}
 807            </fieldset>
 808            <fieldset>
 809              <label for="api_key">API Key</label>
 810              <input type="text" name="api_key" value="${params.value.api_key}" placeholder="Enter API key" oninput=${updateParams} />
 811            </fieldset>
 812          </details>
 813        </form>
 814      `
 815    }
 816
 817    const probColor = (p) => {
 818      const r = Math.floor(192 * (1 - p));
 819      const g = Math.floor(192 * p);
 820      return `rgba(${r},${g},0,0.3)`;
 821    }
 822
 823    const Probabilities = (params) => {
 824      return params.data.map(msg => {
 825        const { completion_probabilities } = msg;
 826        if (
 827          !completion_probabilities ||
 828          completion_probabilities.length === 0
 829        ) return msg.content
 830
 831        if (completion_probabilities.length > 1) {
 832          // Not for byte pair
 833          if (completion_probabilities[0].content.startsWith('byte: \\')) return msg.content
 834
 835          const splitData = completion_probabilities.map(prob => ({
 836            content: prob.content,
 837            completion_probabilities: [prob]
 838          }))
 839          return html`<${Probabilities} data=${splitData} />`
 840        }
 841
 842        const { probs, content } = completion_probabilities[0]
 843        const found = probs.find(p => p.tok_str === msg.content)
 844        const pColor = found ? probColor(found.prob) : 'transparent'
 845
 846        const popoverChildren = html`
 847          <div class="prob-set">
 848            ${probs.map((p, index) => {
 849          return html`
 850                <div
 851                  key=${index}
 852                  title=${`prob: ${p.prob}`}
 853                  style=${{
 854              padding: '0.3em',
 855              backgroundColor: p.tok_str === content ? probColor(p.prob) : 'transparent'
 856            }}
 857                >
 858                  <span>${p.tok_str}: </span>
 859                  <span>${Math.floor(p.prob * 100)}%</span>
 860                </div>
 861              `
 862        })}
 863          </div>
 864        `
 865
 866        return html`
 867          <${Popover} style=${{ backgroundColor: pColor }} popoverChildren=${popoverChildren}>
 868            ${msg.content.match(/\n/gim) ? html`<br />` : msg.content}
 869          </>
 870        `
 871      });
 872    }
 873
 874    // poor mans markdown replacement
 875    const Markdownish = (params) => {
 876      const md = params.text
 877        .replace(/&/g, '&amp;')
 878        .replace(/</g, '&lt;')
 879        .replace(/>/g, '&gt;')
 880        .replace(/^#{1,6} (.*)$/gim, '<h3>$1</h3>')
 881        .replace(/\*\*(.*?)\*\*/g, '<strong>$1</strong>')
 882        .replace(/__(.*?)__/g, '<strong>$1</strong>')
 883        .replace(/\*(.*?)\*/g, '<em>$1</em>')
 884        .replace(/_(.*?)_/g, '<em>$1</em>')
 885        .replace(/```.*?\n([\s\S]*?)```/g, '<pre><code>$1</code></pre>')
 886        .replace(/`(.*?)`/g, '<code>$1</code>')
 887        .replace(/\n/gim, '<br />');
 888      return html`<span dangerouslySetInnerHTML=${{ __html: md }} />`;
 889    };
 890
 891    const ModelGenerationInfo = (params) => {
 892      if (!llamaStats.value) {
 893        return html`<span/>`
 894      }
 895      return html`
 896        <span>
 897          ${llamaStats.value.tokens_predicted} predicted, ${llamaStats.value.tokens_cached} cached, ${llamaStats.value.timings.predicted_per_token_ms.toFixed()}ms per token, ${llamaStats.value.timings.predicted_per_second.toFixed(2)} tokens per second
 898        </span>
 899      `
 900    }
 901
 902    // simple popover impl
 903    const Popover = (props) => {
 904      const isOpen = useSignal(false);
 905      const position = useSignal({ top: '0px', left: '0px' });
 906      const buttonRef = useRef(null);
 907      const popoverRef = useRef(null);
 908
 909      const togglePopover = () => {
 910        if (buttonRef.current) {
 911          const rect = buttonRef.current.getBoundingClientRect();
 912          position.value = {
 913            top: `${rect.bottom + window.scrollY}px`,
 914            left: `${rect.left + window.scrollX}px`,
 915          };
 916        }
 917        isOpen.value = !isOpen.value;
 918      };
 919
 920      const handleClickOutside = (event) => {
 921        if (popoverRef.current && !popoverRef.current.contains(event.target) && !buttonRef.current.contains(event.target)) {
 922          isOpen.value = false;
 923        }
 924      };
 925
 926      useEffect(() => {
 927        document.addEventListener('mousedown', handleClickOutside);
 928        return () => {
 929          document.removeEventListener('mousedown', handleClickOutside);
 930        };
 931      }, []);
 932
 933      return html`
 934        <span style=${props.style} ref=${buttonRef} onClick=${togglePopover}>${props.children}</span>
 935        ${isOpen.value && html`
 936          <${Portal} into="#portal">
 937            <div
 938              ref=${popoverRef}
 939              class="popover-content"
 940              style=${{
 941            top: position.value.top,
 942            left: position.value.left,
 943          }}
 944            >
 945              ${props.popoverChildren}
 946            </div>
 947          </${Portal}>
 948        `}
 949      `;
 950    };
 951
 952    // Source: preact-portal (https://github.com/developit/preact-portal/blob/master/src/preact-portal.js)
 953    /** Redirect rendering of descendants into the given CSS selector */
 954    class Portal extends Component {
 955      componentDidUpdate(props) {
 956        for (let i in props) {
 957          if (props[i] !== this.props[i]) {
 958            return setTimeout(this.renderLayer);
 959          }
 960        }
 961      }
 962
 963      componentDidMount() {
 964        this.isMounted = true;
 965        this.renderLayer = this.renderLayer.bind(this);
 966        this.renderLayer();
 967      }
 968
 969      componentWillUnmount() {
 970        this.renderLayer(false);
 971        this.isMounted = false;
 972        if (this.remote && this.remote.parentNode) this.remote.parentNode.removeChild(this.remote);
 973      }
 974
 975      findNode(node) {
 976        return typeof node === 'string' ? document.querySelector(node) : node;
 977      }
 978
 979      renderLayer(show = true) {
 980        if (!this.isMounted) return;
 981
 982        // clean up old node if moving bases:
 983        if (this.props.into !== this.intoPointer) {
 984          this.intoPointer = this.props.into;
 985          if (this.into && this.remote) {
 986            this.remote = render(html`<${PortalProxy} />`, this.into, this.remote);
 987          }
 988          this.into = this.findNode(this.props.into);
 989        }
 990
 991        this.remote = render(html`
 992          <${PortalProxy} context=${this.context}>
 993            ${show && this.props.children || null}
 994          </${PortalProxy}>
 995        `, this.into, this.remote);
 996      }
 997
 998      render() {
 999        return null;
1000      }
1001    }
1002    // high-order component that renders its first child if it exists.
1003    // used as a conditional rendering proxy.
1004    class PortalProxy extends Component {
1005      getChildContext() {
1006        return this.props.context;
1007      }
1008      render({ children }) {
1009        return children || null;
1010      }
1011    }
1012
1013    function App(props) {
1014      useEffect(() => {
1015        const query = new URLSearchParams(location.search).get("q");
1016        if (query) chat(query);
1017      }, []);
1018
1019      return html`
1020        <div class="mode-${session.value.type}">
1021          <header>
1022            <h1>llama.cpp</h1>
1023          </header>
1024
1025          <section id="write">
1026            <${session.value.type === 'chat' ? MessageInput : CompletionControls} />
1027          </section>
1028
1029          <main id="content">
1030            <${chatStarted.value ? ChatLog : ConfigForm} />
1031          </main>
1032
1033          <footer>
1034            <p><${ModelGenerationInfo} /></p>
1035            <p>Powered by <a href="https://github.com/ggml-org/llama.cpp">llama.cpp</a> and <a href="https://ggml.ai">ggml.ai</a>.</p>
1036          </footer>
1037        </div>
1038      `;
1039    }
1040
1041    render(h(App), document.querySelector('#container'));
1042  </script>
1043</head>
1044
1045<body>
1046  <div id="container">
1047    <input type="file" id="fileInput" accept="image/*" style="display: none;">
1048  </div>
1049  <div id="portal"></div>
1050</body>
1051
1052</html>