1
 2#!/usr/bin/env pwsh
 3
 4# Basedir on device
 5$basedir=".\pkg-snapdragon"
 6
 7$cli_opts=$args
 8
 9$model="Llama-3.2-3B-Instruct-Q4_0.gguf"
10if ($null -ne $env:M) {
11    $model=$env:M
12}
13
14$device="HTP0"
15if ($null -ne $env:D) {
16    $device=$env:D
17}
18
19if ($null -ne $env:V) {
20    $env:GGML_HEXAGON_VERBOSE=$env:V
21}
22
23if ($null -ne $env:E) {
24    $env:GGML_HEXAGON_EXPERIMENTAL=$env:E
25}
26
27if ($null -ne $env:SCHED) {
28    $env:GGML_SCHED_DEBUG=$env:SCHED; $cli_opts="$cli_opts -v"
29}
30
31if ($null -ne $env:PROF) {
32    $env:GGML_HEXAGON_PROFILE=$env:PROF; $env:GGML_HEXAGON_OPSYNC=1
33}
34
35if ($null -ne $env:OPMASK) {
36    $env:GGML_HEXAGON_OPMASK=$env:OPMASK
37}
38
39if ($null -ne $env:NHVX) {
40    $env:GGML_HEXAGON_NHVX=$env:NHVX
41}
42
43if ($null -ne $env:NDEV) {
44    $env:GGML_HEXAGON_NDEV=$env:NDEV
45}
46
47$env:ADSP_LIBRARY_PATH="$basedir\lib"
48
49& "$basedir\bin\llama-completion.exe" `
50    --no-mmap -no-cnv -m $basedir\..\..\gguf\$model `
51    --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 `
52    --ctx-size 8192 --batch-size 128 -ctk q8_0 -ctv q8_0 -fa on `
53    -ngl 99 --device $device $cli_opts