1#!/usr/bin/env bash
 2
 3if [ $# -lt 2 ]; then
 4    echo "usage: ./scripts/compare-commits.sh <commit1> <commit2> [tool] [additional arguments]"
 5    echo "  tool: 'llama-bench' (default) or 'test-backend-ops'"
 6    echo "  additional arguments: passed to the selected tool"
 7    exit 1
 8fi
 9
10set -e
11set -x
12
13# Parse arguments
14commit1=$1
15commit2=$2
16tool=${3:-llama-bench}
17additional_args="${@:4}"
18
19# Validate tool argument
20if [ "$tool" != "llama-bench" ] && [ "$tool" != "test-backend-ops" ]; then
21    echo "Error: tool must be 'llama-bench' or 'test-backend-ops'"
22    exit 1
23fi
24
25# verify at the start that the compare script has all the necessary dependencies installed
26./scripts/compare-llama-bench.py --check
27
28if ! command -v sqlite3 >/dev/null 2>&1; then
29    echo "Error: sqlite3 is not installed or not in PATH"
30    echo "Please install sqlite3 to use this script"
31    exit 1
32fi
33
34if [ "$tool" = "llama-bench" ]; then
35    db_file="llama-bench.sqlite"
36    target="llama-bench"
37    run_args="-o sql -oe md $additional_args"
38else  # test-backend-ops
39    db_file="test-backend-ops.sqlite"
40    target="test-backend-ops"
41    run_args="perf --output sql $additional_args"
42fi
43
44rm -f "$db_file" > /dev/null
45
46# to test a backend, call the script with the corresponding environment variable (e.g. GGML_CUDA=1 ./scripts/compare-commits.sh ...)
47if [ -n "$GGML_CUDA" ]; then
48    CMAKE_OPTS="${CMAKE_OPTS} -DGGML_CUDA=ON"
49fi
50
51dir="build-bench"
52
53function run {
54    rm -fr ${dir} > /dev/null
55    cmake -B ${dir} -S . ${CMAKE_OPTS} > /dev/null
56    cmake --build ${dir} -t $target -j $(nproc) > /dev/null
57    ${dir}/bin/$target $run_args | sqlite3 "$db_file"
58}
59
60git checkout $commit1 > /dev/null
61run
62
63git checkout $commit2 > /dev/null
64run
65
66./scripts/compare-llama-bench.py -b $commit1 -c $commit2 --tool $tool -i "$db_file"