1#!/usr/bin/env bash
2set -e
3
4# Array of models to iterate over
5declare -a params=(
6 "Gemma2ForCausalLM 64"
7 "LlamaForCausalLM 64"
8 "Phi3ForCausalLM 64"
9)
10
11MODELS_REPO=lora-tests
12MODELS_REPO_URL=https://huggingface.co/ggml-org/$MODELS_REPO
13COMMIT=c26d5fb85b4070a9e9c4e65d132c783b98086890
14
15# Clone the Hugging Face repository if the directory does not exist
16if [ ! -d "$MODELS_REPO" ]; then
17 echo "Cloning the Hugging Face repository..."
18 git clone $MODELS_REPO_URL --depth 1
19 cd $MODELS_REPO
20 git fetch --depth=1 origin $COMMIT
21 git reset --hard $COMMIT
22 cd -
23else
24 echo "Repository already exists. Skipping clone."
25fi
26
27# Array to store results to print
28results=()
29
30trim_leading_whitespace() {
31 local input_string="$1"
32 echo "${input_string#"${input_string%%[![:space:]]*}"}"
33}
34
35extract_starting_substring() {
36 local reference_string="$1"
37 local target_string="$2"
38
39 local target_length=${#target_string}
40 echo "${reference_string:0:$target_length}"
41}
42
43get_first_word() {
44 local input_string="$1"
45 read -r first_word _ <<< "$input_string"
46 echo "$first_word"
47}
48
49# Load the expected strings
50EXPECTED_BASE_FULL=$(cat $MODELS_REPO/data/pale_blue_dot.txt)
51EXPECTED_LORA_FULL=$(cat $MODELS_REPO/data/bohemian_rhapsody.txt)
52EXPECTED_BASE_FIRST_WORD=$(get_first_word "$EXPECTED_BASE_FULL")
53EXPECTED_LORA_FIRST_WORD=$(get_first_word "$EXPECTED_LORA_FULL")
54
55run_conversion_and_inference_lora() {
56 local model_name=$1
57 local hidden_size=$2
58
59 echo -e "\n\n-------- RUNNING TEST FOR MODEL $model_name --------\n\n"
60
61 # Convert safetensors to gguf
62 echo "Running convert_hf_to_gguf.py for $model_name with hidden_size $hidden_size..."
63 python convert_hf_to_gguf.py $MODELS_REPO/$model_name/hidden_size=$hidden_size/base \
64 --outfile $MODELS_REPO/$model_name/hidden_size=$hidden_size/base/Base-F32.gguf \
65 --outtype f32
66
67 echo -e "\n\n---------------------------\n\n"
68 echo "Running convert_lora_to_gguf.py for $model_name with hidden_size $hidden_size..."
69 python3 convert_lora_to_gguf.py $MODELS_REPO/$model_name/hidden_size=$hidden_size/lora \
70 --base $MODELS_REPO/$model_name/hidden_size=$hidden_size/base \
71 --outtype f32
72
73 echo -e "\n\n---------------------------\n\n"
74 echo "Running llama-export-lora with lora for $model_name with hidden_size $hidden_size..."
75 ./llama-export-lora \
76 -m $MODELS_REPO/$model_name/hidden_size=$hidden_size/base/Base-F32.gguf \
77 -o $MODELS_REPO/$model_name/hidden_size=$hidden_size/base/Base-F32-lora-merged.gguf \
78 --lora $MODELS_REPO/$model_name/hidden_size=$hidden_size/lora/Lora-F32-LoRA.gguf
79
80 # Run inference
81 echo -e "\n\n---------------------------\n\n"
82 echo "Running llama-completion without lora for $model_name with hidden_size $hidden_size..."
83 OUTPUT_BASE=$(./llama-completion -no-cnv -m $MODELS_REPO/$model_name/hidden_size=$hidden_size/base/Base-F32.gguf \
84 -p "$EXPECTED_BASE_FIRST_WORD" -n 50 --seed 42 --temp 0)
85
86 echo -e "\n\n---------------------------\n\n"
87 echo "Running llama-completion with hot lora for $model_name with hidden_size $hidden_size..."
88 OUTPUT_LORA_HOT=$(./llama-completion -no-cnv -m $MODELS_REPO/$model_name/hidden_size=$hidden_size/base/Base-F32.gguf \
89 --lora $MODELS_REPO/$model_name/hidden_size=$hidden_size/lora/Lora-F32-LoRA.gguf \
90 -p "$EXPECTED_LORA_FIRST_WORD" -n 50 --seed 42 --temp 0)
91
92 echo -e "\n\n---------------------------\n\n"
93 echo "Running llama-completion with merged lora for $model_name with hidden_size $hidden_size..."
94 OUTPUT_LORA_MERGED=$(./llama-completion -no-cnv -m $MODELS_REPO/$model_name/hidden_size=$hidden_size/base/Base-F32-lora-merged.gguf \
95 -p "$EXPECTED_LORA_FIRST_WORD" -n 50 --seed 42 --temp 0)
96
97 # Remove any initial white space
98 OUTPUT_BASE=$(trim_leading_whitespace "$OUTPUT_BASE")
99 OUTPUT_LORA_HOT=$(trim_leading_whitespace "$OUTPUT_LORA_HOT")
100 OUTPUT_LORA_MERGED=$(trim_leading_whitespace "$OUTPUT_LORA_MERGED")
101 # Extract the corresponding substring from full string
102 EXPECTED_BASE=$(extract_starting_substring "$EXPECTED_BASE_FULL" "$OUTPUT_BASE")
103 EXPECTED_LORA=$(extract_starting_substring "$EXPECTED_LORA_FULL" "$OUTPUT_LORA_HOT")
104
105 # Assert output equals the expected output
106 if [[ "$OUTPUT_BASE" != "$EXPECTED_BASE" ]]; then
107 echo "Error: $model_name OUTPUT_BASE does not start with the expected string."
108 echo -e "Out=$OUTPUT_BASE\n\nExp=$EXPECTED_BASE"
109 exit 1
110 fi
111 if [[ "$OUTPUT_LORA_HOT" != "$EXPECTED_LORA" ]]; then
112 echo "Error: $model_name OUTPUT_LORA_HOT does not start with the expected string."
113 echo -e "Out=$OUTPUT_LORA_HOT\n\nExp=$EXPECTED_LORA"
114 exit 1
115 fi
116 if [[ "$OUTPUT_LORA_MERGED" != "$EXPECTED_LORA" ]]; then
117 echo "Error: $model_name OUTPUT_LORA_MERGED does not start with the expected string."
118 echo -e "Out=$OUTPUT_LORA_MERGED\n\nExp=$EXPECTED_LORA"
119 exit 1
120 fi
121
122 # Store the results
123 results+=("
124 \n\033[1mResults for $model_name with hidden_size $hidden_size:\033[0m
125 \n\033[32m • Base:\n$OUTPUT_BASE
126 \n\033[34m • Lora hot:\n$OUTPUT_LORA_HOT
127 \n\033[36m • Lora merged:\n$OUTPUT_LORA_MERGED
128 \n \033[0m
129 ")
130
131 echo "All tests passed for $model_name with hidden_size $hidden_size!"
132}
133
134# Run test for each model
135for param in "${params[@]}"; do
136 run_conversion_and_inference_lora $param
137done
138
139# Print results
140echo -e "\n\n---------------------------\n\n"
141echo -e "\n\033[1mSummary of All Results:\033[0m"
142for result in "${results[@]}"; do
143 echo -e "$result"
144done