1name: Check Pre-Tokenizer Hashes
 2
 3on:
 4    push:
 5        paths:
 6            - 'convert_hf_to_gguf.py'
 7            - 'convert_hf_to_gguf_update.py'
 8    pull_request:
 9        paths:
10            - 'convert_hf_to_gguf.py'
11            - 'convert_hf_to_gguf_update.py'
12
13jobs:
14    pre-tokenizer-hashes:
15        runs-on: ubuntu-slim
16
17        steps:
18        - name: Checkout repository
19          uses: actions/checkout@v6
20
21        - name: Set up Python
22          uses: actions/setup-python@v6
23          with:
24              python-version: '3.11'
25
26        - name: Install Python dependencies
27          run: |
28              python3 -m venv .venv
29              .venv/bin/pip install -r requirements/requirements-convert_hf_to_gguf_update.txt
30
31        - name: Update pre-tokenizer hashes
32          run: |
33              cp convert_hf_to_gguf.py /tmp
34              .venv/bin/python convert_hf_to_gguf_update.py --check-missing
35
36        - name: Check if committed pre-tokenizer hashes matches generated version
37          run: |
38              if ! diff -q convert_hf_to_gguf.py /tmp/convert_hf_to_gguf.py; then
39                  echo "Model pre-tokenizer hashes (in convert_hf_to_gguf.py) do not match generated hashes (from convert_hf_to_gguf_update.py)."
40                  echo "To fix: run ./convert_hf_to_gguf_update.py and commit the updated convert_hf_to_gguf.py along with your changes"
41                  echo "Differences found:"
42                  diff convert_hf_to_gguf.py /tmp/convert_hf_to_gguf.py || true
43                  exit 1
44              fi
45              echo "Model pre-tokenizer hashes are up to date."