1#!/usr/bin/env bash
 2
 3API_URL="${API_URL:-http://127.0.0.1:8080}"
 4
 5CHAT=(
 6    "Hello, Assistant."
 7    "Hello. How may I help you today?"
 8    "Please tell me the largest city in Europe."
 9    "Sure. The largest city in Europe is Moscow, the capital of Russia."
10)
11
12INSTRUCTION="A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions."
13
14trim() {
15    shopt -s extglob
16    set -- "${1##+([[:space:]])}"
17    printf "%s" "${1%%+([[:space:]])}"
18}
19
20trim_trailing() {
21    shopt -s extglob
22    printf "%s" "${1%%+([[:space:]])}"
23}
24
25format_prompt() {
26    echo -n "${INSTRUCTION}"
27    printf "\n### Human: %s\n### Assistant: %s" "${CHAT[@]}" "$1"
28}
29
30tokenize() {
31    curl \
32        --silent \
33        --request POST \
34        --url "${API_URL}/tokenize" \
35        --header "Content-Type: application/json" \
36        --data-raw "$(jq -ns --arg content "$1" '{content:$content}')" \
37    | jq '.tokens[]'
38}
39
40N_KEEP=$(tokenize "${INSTRUCTION}" | wc -l)
41
42chat_completion() {
43    PROMPT="$(trim_trailing "$(format_prompt "$1")")"
44    DATA="$(echo -n "$PROMPT" | jq -Rs --argjson n_keep $N_KEEP '{
45        prompt: .,
46        temperature: 0.2,
47        top_k: 40,
48        top_p: 0.9,
49        n_keep: $n_keep,
50        n_predict: 256,
51        cache_prompt: true,
52        stop: ["\n### Human:"],
53        stream: true
54    }')"
55
56    ANSWER=''
57
58    while IFS= read -r LINE; do
59        if [[ $LINE = data:* ]]; then
60            CONTENT="$(echo "${LINE:5}" | jq -r '.content')"
61            printf "%s" "${CONTENT}"
62            ANSWER+="${CONTENT}"
63        fi
64    done < <(curl \
65        --silent \
66        --no-buffer \
67        --request POST \
68        --url "${API_URL}/completion" \
69        --header "Content-Type: application/json" \
70        --data-raw "${DATA}")
71
72    printf "\n"
73
74    CHAT+=("$1" "$(trim "$ANSWER")")
75}
76
77while true; do
78    read -r -e -p "> " QUESTION
79    chat_completion "${QUESTION}"
80done