1#!/usr/bin/env bash
  2set -euo pipefail
  3
  4#
  5# check-requirements.sh checks all requirements files for each top-level
  6# convert*.py script.
  7#
  8# WARNING: This is quite IO intensive, because a fresh venv is set up for every
  9# python script. As of 2023-12-22, this writes ~2.7GB of data. An adequately
 10# sized tmpfs /tmp or ramdisk is recommended if running this frequently.
 11#
 12# usage:    check-requirements.sh [<working_dir>]
 13#           check-requirements.sh nocleanup [<working_dir>]
 14#
 15# where:
 16#           - <working_dir> is a directory that can be used as the base for
 17#               setting up the venvs. Defaults to `/tmp`.
 18#           - 'nocleanup' as the first argument will disable automatic cleanup
 19#               of the files created by this script.
 20#
 21# requires:
 22#           - bash >= 3.2.57
 23#           - shellcheck
 24#
 25# For each script, it creates a fresh venv, `pip install`s the requirements, and
 26# finally imports the python script to check for `ImportError`.
 27#
 28
 29log() {
 30    local level=$1 msg=$2
 31    printf >&2 '%s: %s\n' "$level" "$msg"
 32}
 33
 34debug() {
 35    log DEBUG "$@"
 36}
 37
 38info() {
 39    log INFO "$@"
 40}
 41
 42fatal() {
 43    log FATAL "$@"
 44    exit 1
 45}
 46
 47cleanup() {
 48    if [[ -n ${workdir+x} && -d $workdir && -w $workdir ]]; then
 49        info "Removing $workdir"
 50        local count=0
 51        rm -rfv -- "$workdir" | while read -r; do
 52            if (( count++ > 750 )); then
 53                printf .
 54                count=0
 55            fi
 56        done
 57        printf '\n'
 58        info "Removed $workdir"
 59    fi
 60}
 61
 62do_cleanup=1
 63if [[ ${1-} == nocleanup ]]; then
 64    do_cleanup=0; shift
 65fi
 66
 67if (( do_cleanup )); then
 68    trap exit INT TERM
 69    trap cleanup EXIT
 70fi
 71
 72this=$(realpath -- "$0"); readonly this
 73cd "$(dirname "$this")/.." # PWD should stay in llama.cpp project directory
 74
 75shellcheck "$this"
 76
 77readonly reqs_dir=requirements
 78
 79if [[ ${1+x} ]]; then
 80    tmp_dir=$(realpath -- "$1")
 81    if [[ ! ( -d $tmp_dir && -w $tmp_dir ) ]]; then
 82        fatal "$tmp_dir is not a writable directory"
 83    fi
 84else
 85    tmp_dir=/tmp
 86fi
 87
 88workdir=$(mktemp -d "$tmp_dir/check-requirements.XXXX"); readonly workdir
 89info "Working directory: $workdir"
 90
 91check_requirements() {
 92    local reqs=$1
 93
 94    info "$reqs: beginning check"
 95    pip --disable-pip-version-check install -qr "$reqs"
 96    info "$reqs: OK"
 97}
 98
 99check_convert_script() {
100    local py=$1             # e.g. ./convert_hf_to_gguf.py
101    local pyname=${py##*/}  # e.g. convert_hf_to_gguf.py
102    pyname=${pyname%.py}    # e.g. convert_hf_to_gguf
103
104    info "$py: beginning check"
105
106    local reqs="$reqs_dir/requirements-$pyname.txt"
107    if [[ ! -r $reqs ]]; then
108        fatal "$py missing requirements. Expected: $reqs"
109    fi
110
111    # Check that all sub-requirements are added to top-level requirements.txt
112    if ! grep -qF "$reqs" requirements.txt; then
113        fatal "$reqs needs to be added to requirements.txt"
114    fi
115
116    local venv="$workdir/$pyname-venv"
117    python3 -m venv "$venv"
118
119    (
120        # shellcheck source=/dev/null
121        source "$venv/bin/activate"
122
123        check_requirements "$reqs"
124
125        python - "$py" "$pyname" <<'EOF'
126import sys
127from importlib.machinery import SourceFileLoader
128py, pyname = sys.argv[1:]
129SourceFileLoader(pyname, py).load_module()
130EOF
131    )
132
133    if (( do_cleanup )); then
134        rm -rf -- "$venv"
135    fi
136
137    info "$py: imports OK"
138}
139
140readonly ignore_eq_eq='check_requirements: ignore "=="'
141
142for req in */**/requirements*.txt; do
143    # Make sure exact release versions aren't being pinned in the requirements
144    # Filters out the ignore string
145    if grep -vF "$ignore_eq_eq" "$req" | grep -q '=='; then
146        tab=$'\t'
147        cat >&2 <<EOF
148FATAL: Avoid pinning exact package versions. Use '~=' instead.
149You can suppress this error by appending the following to the line:
150$tab# $ignore_eq_eq
151EOF
152        exit 1
153    fi
154done
155
156all_venv="$workdir/all-venv"
157python3 -m venv "$all_venv"
158
159(
160    # shellcheck source=/dev/null
161    source "$all_venv/bin/activate"
162    check_requirements requirements.txt
163)
164
165if (( do_cleanup )); then
166    rm -rf -- "$all_venv"
167fi
168
169check_convert_script examples/convert_legacy_llama.py
170for py in convert_*.py; do
171    # skip convert_hf_to_gguf_update.py
172    # TODO: the check is failing for some reason:
173    #       https://github.com/ggml-org/llama.cpp/actions/runs/8875330981/job/24364557177?pr=6920
174    [[ $py == convert_hf_to_gguf_update.py ]] && continue
175
176    check_convert_script "$py"
177done
178
179info 'Done! No issues found.'