1#!/usr/bin/env bash
 2
 3if [ $# -lt 2 ]; then
 4    printf "Usage: $0 <git-repo> <target-folder> [<test-exe>]\n"
 5    exit 1
 6fi
 7
 8if [ $# -eq 3 ]; then
 9    toktest=$3
10else
11    toktest="./test-tokenizer-0"
12fi
13
14if [ ! -x $toktest ]; then
15    printf "Test executable \"$toktest\" not found!\n"
16    exit 1
17fi
18
19repo=$1
20folder=$2
21
22if [ -d $folder ] && [ -d $folder/.git ]; then
23    (cd $folder; git pull)
24else
25    git clone $repo $folder
26
27    # byteswap models if on big endian
28    if [ "$(uname -m)" = s390x ]; then
29        for f in $folder/*/*.gguf; do
30            echo YES | python3 "$(dirname $0)/../gguf-py/gguf/scripts/gguf_convert_endian.py" $f big
31        done
32    fi
33fi
34
35shopt -s globstar
36for gguf in $folder/**/*.gguf; do
37    if [ -f $gguf.inp ] && [ -f $gguf.out ]; then
38        $toktest $gguf
39    else
40        printf "Found \"$gguf\" without matching inp/out files, ignoring...\n"
41    fi
42done
43