1#!/usr/bin/env bash
  2#
  3# Synchronize ggml changes to llama.cpp
  4#
  5# Usage:
  6#
  7#   $ cd /path/to/llama.cpp
  8#   $ ./scripts/sync-ggml-am.sh -skip hash0,hash1,hash2... -C 3
  9#
 10
 11set -e
 12
 13sd=$(dirname $0)
 14cd $sd/../
 15
 16SRC_LLAMA=$(pwd)
 17SRC_GGML=$(cd ../ggml; pwd)
 18
 19if [ ! -d $SRC_GGML ]; then
 20    echo "ggml not found at $SRC_GGML"
 21    exit 1
 22fi
 23
 24lc=$(cat $SRC_LLAMA/scripts/sync-ggml.last)
 25echo "Syncing ggml changes since commit $lc"
 26
 27to_skip=""
 28
 29# context for git patches in number of lines
 30ctx="8"
 31
 32while [ "$1" != "" ]; do
 33    case $1 in
 34        -skip )
 35            shift
 36            to_skip=$1
 37            ;;
 38        -C )
 39            shift
 40            ctx=$1
 41            ;;
 42    esac
 43    shift
 44done
 45
 46cd $SRC_GGML
 47
 48git log --oneline $lc..HEAD
 49git log --oneline $lc..HEAD --reverse | grep -v "(llama/[0-9]*)" | cut -d' ' -f1 > $SRC_LLAMA/ggml-commits
 50
 51if [ ! -s $SRC_LLAMA/ggml-commits ]; then
 52    rm -v $SRC_LLAMA/ggml-commits
 53    echo "No new commits"
 54    exit 0
 55fi
 56
 57if [ -f $SRC_LLAMA/ggml-src.patch ]; then
 58    rm -v $SRC_LLAMA/ggml-src.patch
 59fi
 60
 61while read c; do
 62    if [ -n "$to_skip" ]; then
 63        if [[ $to_skip == *"$c"* ]]; then
 64            echo "Skipping $c"
 65            continue
 66        fi
 67    fi
 68
 69    git format-patch -U${ctx} -k $c~1..$c --stdout -- \
 70        CMakeLists.txt \
 71        src/CMakeLists.txt \
 72        cmake/BuildTypes.cmake \
 73        cmake/GitVars.cmake \
 74        cmake/common.cmake \
 75        cmake/ggml-config.cmake.in \
 76        src/ggml-cpu/cmake/FindSIMD.cmake \
 77        src/ggml* \
 78        include/ggml*.h \
 79        include/gguf*.h \
 80        tests/test-opt.cpp \
 81        tests/test-quantize-fns.cpp \
 82        tests/test-quantize-perf.cpp \
 83        tests/test-backend-ops.cpp \
 84        LICENSE \
 85        scripts/gen-authors.sh \
 86        >> $SRC_LLAMA/ggml-src.patch
 87done < $SRC_LLAMA/ggml-commits
 88
 89rm -v $SRC_LLAMA/ggml-commits
 90
 91# delete files if empty
 92if [ ! -s $SRC_LLAMA/ggml-src.patch ]; then
 93    rm -v $SRC_LLAMA/ggml-src.patch
 94fi
 95
 96cd $SRC_LLAMA
 97
 98if [ -f $SRC_LLAMA/ggml-src.patch ]; then
 99    # replace PR numbers
100    #
101    # Subject: some text (#1234)
102    # Subject: some text (ggml/1234)
103    cat ggml-src.patch | sed -e 's/^Subject: \(.*\) (#\([0-9]*\))/Subject: \1 (ggml\/\2)/' > ggml-src.patch.tmp
104    mv ggml-src.patch.tmp ggml-src.patch
105
106    cat ggml-src.patch | sed -e 's/^\(.*\) (#\([0-9]*\))$/\1 (ggml\/\2)/' > ggml-src.patch.tmp
107    mv ggml-src.patch.tmp ggml-src.patch
108
109    # replace filenames:
110    #
111    # CMakelists.txt       -> ggml/CMakeLists.txt
112    # src/CMakeLists.txt   -> ggml/src/CMakeLists.txt
113
114    # cmake/BuildTypes.cmake            -> ggml/cmake/BuildTypes.cmake
115    # cmake/GitVars.cmake               -> ggml/cmake/GitVars.cmake
116    # cmake/common.cmake                -> ggml/cmake/common.cmake
117    # cmake/ggml-config.cmake.in        -> ggml/cmake/ggml-config.cmake.in
118    # src/ggml-cpu/cmake/FindSIMD.cmake -> ggml/src/ggml-cpu/cmake/FindSIMD.cmake
119    #
120    # src/ggml* -> ggml/src/ggml*
121    #
122    # include/ggml*.h -> ggml/include/ggml*.h
123    # include/gguf*.h -> ggml/include/gguf*.h
124    #
125    # tests/test*.cpp -> tests/
126    #
127    # LICENSE                -> LICENSE
128    # scripts/gen-authors.sh -> scripts/gen-authors.sh
129
130    cat ggml-src.patch | sed -E \
131        -e 's/([[:space:]]| [ab]\/)CMakeLists.txt/\1ggml\/CMakeLists.txt/g' \
132        -e 's/([[:space:]]| [ab]\/)src\/CMakeLists.txt/\1ggml\/src\/CMakeLists.txt/g' \
133        -e 's/([[:space:]]| [ab]\/)cmake\/BuildTypes.cmake/\1ggml\/cmake\/BuildTypes.cmake/g' \
134        -e 's/([[:space:]]| [ab]\/)cmake\/GitVars.cmake/\1ggml\/cmake\/GitVars.cmake/g' \
135        -e 's/([[:space:]]| [ab]\/)cmake\/common.cmake/\1ggml\/cmake\/common.cmake/g' \
136        -e 's/([[:space:]]| [ab]\/)cmake\/ggml-config.cmake.in/\1ggml\/cmake\/ggml-config.cmake.in/g' \
137        -e 's/([[:space:]]| [ab]\/)src\/ggml-cpu\/cmake\/FindSIMD.cmake/\1ggml\/src\/ggml-cpu\/cmake\/FindSIMD.cmake/g' \
138        -e 's/([[:space:]]| [ab]\/)src\/ggml(.*)/\1ggml\/src\/ggml\2/g' \
139        -e 's/([[:space:]]| [ab]\/)include\/ggml(.*)\.h/\1ggml\/include\/ggml\2.h/g' \
140        -e 's/([[:space:]]| [ab]\/)include\/gguf(.*)\.h/\1ggml\/include\/gguf\2.h/g' \
141        -e 's/([[:space:]]| [ab]\/)tests\/(.*)\.cpp/\1tests\/\2.cpp/g' \
142        -e 's/([[:space:]]| [ab]\/)LICENSE/\1LICENSE/g' \
143        -e 's/([[:space:]]| [ab]\/)scripts\/gen-authors\.sh/\1scripts\/gen-authors.sh/g' \
144        > ggml-src.patch.tmp
145    mv ggml-src.patch.tmp ggml-src.patch
146
147    git am -C${ctx} ggml-src.patch
148
149    rm -v $SRC_LLAMA/ggml-src.patch
150fi
151
152# update last commit
153cd $SRC_GGML
154git log -1 --format=%H > $SRC_LLAMA/scripts/sync-ggml.last
155
156echo "Done"
157
158exit 0