1name: CI
   2
   3on:
   4  workflow_dispatch: # allows manual triggering
   5  push:
   6    branches:
   7      - master
   8    paths: [
   9      '.github/workflows/build.yml',
  10      '.github/workflows/build-linux-cross.yml',
  11      '.github/workflows/build-cmake-pkg.yml',
  12      '**/CMakeLists.txt',
  13      '**/.cmake',
  14      '**/*.h',
  15      '**/*.hpp',
  16      '**/*.c',
  17      '**/*.cpp',
  18      '**/*.cu',
  19      '**/*.cuh',
  20      '**/*.swift',
  21      '**/*.m',
  22      '**/*.metal',
  23      '**/*.comp',
  24      '**/*.glsl',
  25      '**/*.wgsl'
  26    ]
  27
  28  pull_request:
  29    types: [opened, synchronize, reopened]
  30    paths: [
  31      '.github/workflows/build.yml',
  32      '.github/workflows/build-linux-cross.yml',
  33      '.github/workflows/build-cmake-pkg.yml',
  34      '**/CMakeLists.txt',
  35      '**/.cmake',
  36      '**/*.h',
  37      '**/*.hpp',
  38      '**/*.c',
  39      '**/*.cpp',
  40      '**/*.cu',
  41      '**/*.cuh',
  42      '**/*.swift',
  43      '**/*.m',
  44      '**/*.metal',
  45      '**/*.comp',
  46      '**/*.glsl',
  47      '**/*.wgsl'
  48    ]
  49
  50concurrency:
  51  group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
  52  cancel-in-progress: true
  53
  54env:
  55  GGML_NLOOP: 3
  56  GGML_N_THREADS: 1
  57  LLAMA_LOG_COLORS: 1
  58  LLAMA_LOG_PREFIX: 1
  59  LLAMA_LOG_TIMESTAMPS: 1
  60
  61jobs:
  62  macOS-latest-cmake-arm64:
  63    runs-on: macos-latest
  64
  65    steps:
  66      - name: Clone
  67        id: checkout
  68        uses: actions/checkout@v6
  69
  70      - name: ccache
  71        uses: ggml-org/ccache-action@v1.2.16
  72        with:
  73          key: macOS-latest-cmake-arm64
  74          evict-old-files: 1d
  75          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
  76
  77      - name: Build
  78        id: cmake_build
  79        run: |
  80          sysctl -a
  81          cmake -B build \
  82            -DCMAKE_BUILD_RPATH="@loader_path" \
  83            -DLLAMA_FATAL_WARNINGS=ON \
  84            -DLLAMA_BUILD_BORINGSSL=ON \
  85            -DGGML_METAL_USE_BF16=ON \
  86            -DGGML_METAL_EMBED_LIBRARY=OFF \
  87            -DGGML_METAL_SHADER_DEBUG=ON \
  88            -DGGML_RPC=ON
  89          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
  90          leaks -atExit -- ./build/bin/test-thread-safety -hf ggml-org/gemma-3-270m-qat-GGUF -ngl 99 -p "$(printf 'hello %.0s' {1..128})" -n 16 -c 512 -ub 32 -np 2 -t 2 -lv 1
  91
  92      - name: Test
  93        id: cmake_test
  94        run: |
  95          cd build
  96          ctest -L main --verbose --timeout 900
  97
  98  macOS-latest-cmake-x64:
  99    runs-on: macos-15-intel
 100
 101    steps:
 102      - name: Clone
 103        id: checkout
 104        uses: actions/checkout@v6
 105
 106      - name: ccache
 107        uses: ggml-org/ccache-action@v1.2.16
 108        with:
 109          key: macOS-latest-cmake-x64
 110          evict-old-files: 1d
 111          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 112
 113      - name: Build
 114        id: cmake_build
 115        run: |
 116          sysctl -a
 117          # Metal is disabled due to intermittent failures with Github runners not having a GPU:
 118          # https://github.com/ggml-org/llama.cpp/actions/runs/8635935781/job/23674807267#step:5:2313
 119          cmake -B build \
 120            -DCMAKE_BUILD_RPATH="@loader_path" \
 121            -DLLAMA_FATAL_WARNINGS=ON \
 122            -DLLAMA_BUILD_BORINGSSL=ON \
 123            -DGGML_METAL=OFF \
 124            -DGGML_RPC=ON \
 125            -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
 126          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
 127
 128      - name: Test
 129        id: cmake_test
 130        run: |
 131          cd build
 132          ctest -L main --verbose --timeout 900
 133
 134  macOS-latest-cmake-arm64-webgpu:
 135    runs-on: macos-latest
 136
 137    steps:
 138      - name: Clone
 139        id: checkout
 140        uses: actions/checkout@v6
 141
 142      - name: ccache
 143        uses: ggml-org/ccache-action@v1.2.16
 144        with:
 145          key: macOS-latest-cmake-arm64-webgpu
 146          evict-old-files: 1d
 147          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 148
 149      - name: Dawn Dependency
 150        id: dawn-depends
 151        run: |
 152          DAWN_VERSION="v2.0.0"
 153          DAWN_OWNER="reeselevine"
 154          DAWN_REPO="dawn"
 155          DAWN_ASSET_NAME="Dawn-5e9a4865b1635796ccc77dd30057f2b4002a1355-macos-latest-Release"
 156          echo "Fetching release asset from https://github.com/${DAWN_OWNER}/${DAWN_REPO}/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.zip"
 157          curl -L -o artifact.zip \
 158            "https://github.com/${DAWN_OWNER}/${DAWN_REPO}/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.zip"
 159          mkdir dawn
 160          unzip artifact.zip
 161          tar -xvf ${DAWN_ASSET_NAME}.tar.gz -C dawn --strip-components=1
 162
 163      - name: Build
 164        id: cmake_build
 165        run: |
 166          export CMAKE_PREFIX_PATH=dawn
 167          cmake -B build -DGGML_WEBGPU=ON -DGGML_METAL=OFF -DGGML_BLAS=OFF
 168          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
 169
 170      - name: Test
 171        id: cmake_test
 172        run: |
 173          cd build
 174          ctest -L main --verbose --timeout 900
 175
 176  ubuntu-cpu-cmake:
 177    strategy:
 178      matrix:
 179        include:
 180          - build: 'x64'
 181            os: ubuntu-22.04
 182          - build: 'arm64'
 183            os: ubuntu-22.04-arm
 184          - build: 's390x'
 185            os: ubuntu-24.04-s390x
 186          - build: 'ppc64le'
 187            os: ubuntu-24.04-ppc64le
 188
 189    runs-on: ${{ matrix.os }}
 190
 191    steps:
 192      - name: Clone
 193        id: checkout
 194        uses: actions/checkout@v6
 195
 196      - name: ccache
 197        uses: ggml-org/ccache-action@v1.2.16
 198        with:
 199          key: ubuntu-cpu-cmake-${{ matrix.build }}
 200          evict-old-files: 1d
 201          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 202
 203      - name: Build Dependencies
 204        id: build_depends
 205        run: |
 206          sudo apt-get update
 207          sudo apt-get install -y --no-install-recommends \
 208            python3 python3-pip python3-dev \
 209            libjpeg-dev build-essential libssl-dev \
 210            git-lfs
 211
 212      - name: Python Dependencies
 213        id: python_depends
 214        run: |
 215          python3 -m pip install --upgrade pip
 216          pip3 install ./gguf-py
 217
 218      - name: Swap Endianness
 219        id: endianness
 220        if: ${{ matrix.build == 's390x' }}
 221        run: |
 222          for f in models/*.gguf; do
 223            echo YES | python3 gguf-py/gguf/scripts/gguf_convert_endian.py $f big
 224          done
 225
 226      - name: Build
 227        id: cmake_build
 228        run: |
 229          cmake -B build \
 230            -DLLAMA_FATAL_WARNINGS=ON \
 231            -DGGML_RPC=ON
 232          cmake --build build --config Release -j $(nproc)
 233
 234      - name: Test
 235        id: cmake_test
 236        run: |
 237          cd build
 238          ctest -L main --verbose --timeout 900
 239
 240      - name: Test llama2c conversion
 241        id: llama2c_test
 242        if: ${{ matrix.build != 's390x' }}
 243        run: |
 244          cd build
 245          echo "Fetch tokenizer"
 246          wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/tok512.bin
 247          echo "Fetch llama2c model"
 248          wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin
 249          ./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf
 250          ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
 251
 252      - name: Test llama2c (s390x)
 253        id: llama2c_test_s390x
 254        if: ${{ matrix.build == 's390x' }}
 255        run: |
 256          cd build
 257          echo "Fetch llama2c big-endian model"
 258          wget https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K-be.gguf
 259          ./bin/llama-completion -m stories260K-be.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
 260
 261  ubuntu-latest-cmake-sanitizer:
 262    runs-on: ubuntu-latest
 263
 264    continue-on-error: true
 265
 266    strategy:
 267      matrix:
 268        sanitizer: [ADDRESS, THREAD, UNDEFINED]
 269        build_type: [Debug]
 270
 271    steps:
 272      - name: Clone
 273        id: checkout
 274        uses: actions/checkout@v6
 275
 276      - name: ccache
 277        uses: ggml-org/ccache-action@v1.2.16
 278        with:
 279          key: ubuntu-latest-cmake-sanitizer-${{ matrix.sanitizer }}
 280          evict-old-files: 1d
 281          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 282
 283      - name: Dependencies
 284        id: depends
 285        run: |
 286          sudo apt-get update
 287          sudo apt-get install build-essential libssl-dev
 288
 289      - name: Build
 290        id: cmake_build
 291        if: ${{ matrix.sanitizer != 'THREAD' }}
 292        run: |
 293          cmake -B build \
 294            -DLLAMA_FATAL_WARNINGS=ON \
 295            -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
 296            -DGGML_SANITIZE_${{ matrix.sanitizer }}=ON \
 297            -DCMAKE_BUILD_TYPE=${{ matrix.build_type }}
 298
 299          cmake --build build --config ${{ matrix.build_type }} -j $(nproc)
 300
 301      - name: Build (no OpenMP)
 302        id: cmake_build_no_openmp
 303        if: ${{ matrix.sanitizer == 'THREAD' }}
 304        run: |
 305          cmake -B build \
 306            -DLLAMA_FATAL_WARNINGS=ON \
 307            -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
 308            -DGGML_SANITIZE_${{ matrix.sanitizer }}=ON \
 309            -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
 310            -DGGML_OPENMP=OFF
 311
 312          cmake --build build --config ${{ matrix.build_type }} -j $(nproc)
 313
 314      - name: Test
 315        id: cmake_test
 316        run: |
 317          cd build
 318          ctest -L main --verbose --timeout 900
 319
 320  ubuntu-latest-llguidance:
 321    runs-on: ubuntu-latest
 322
 323    steps:
 324      - name: Clone
 325        id: checkout
 326        uses: actions/checkout@v6
 327
 328      - name: Dependencies
 329        id: depends
 330        run: |
 331          sudo apt-get update
 332          sudo apt-get install build-essential libssl-dev
 333
 334      - name: Build
 335        id: cmake_build
 336        run: |
 337          cmake -B build \
 338            -DLLAMA_FATAL_WARNINGS=ON \
 339            -DLLAMA_LLGUIDANCE=ON
 340          cmake --build build --config Release -j $(nproc)
 341
 342      - name: Test
 343        id: cmake_test
 344        run: |
 345          cd build
 346          ctest -L main --verbose --timeout 900
 347
 348  ubuntu-latest-cmake-rpc:
 349    runs-on: ubuntu-latest
 350
 351    continue-on-error: true
 352
 353    steps:
 354      - name: Clone
 355        id: checkout
 356        uses: actions/checkout@v6
 357
 358      # - name: ccache
 359      #   uses: ggml-org/ccache-action@v1.2.16
 360      #   with:
 361      #     key: ubuntu-latest-cmake-rpc
 362      #     evict-old-files: 1d
 363
 364      - name: Dependencies
 365        id: depends
 366        run: |
 367          sudo apt-get update
 368          sudo apt-get install build-essential libssl-dev
 369
 370      - name: Build
 371        id: cmake_build
 372        run: |
 373          cmake -B build \
 374            -DGGML_RPC=ON
 375          cmake --build build --config Release -j $(nproc)
 376
 377      - name: Test
 378        id: cmake_test
 379        run: |
 380          cd build
 381          ctest -L main --verbose
 382
 383  ubuntu-24-cmake-vulkan-deb:
 384    runs-on: ubuntu-24.04
 385
 386    steps:
 387      - name: Clone
 388        id: checkout
 389        uses: actions/checkout@v6
 390
 391      - name: ccache
 392        uses: ggml-org/ccache-action@v1.2.16
 393        with:
 394          key: ubuntu-24-cmake-vulkan-deb
 395          evict-old-files: 1d
 396          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 397
 398      - name: Dependencies
 399        id: depends
 400        run: |
 401          sudo apt-get install -y glslc libvulkan-dev libssl-dev
 402
 403      - name: Configure
 404        id: cmake_configure
 405        run: |
 406          cmake -B build \
 407            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
 408            -DGGML_BACKEND_DL=ON \
 409            -DGGML_CPU_ALL_VARIANTS=ON \
 410            -DGGML_VULKAN=ON
 411
 412      - name: Build
 413        id: cmake_build
 414        run: |
 415          cmake --build build -j $(nproc)
 416
 417  ubuntu-24-cmake-vulkan:
 418    runs-on: ubuntu-24.04
 419
 420    steps:
 421      - name: Clone
 422        id: checkout
 423        uses: actions/checkout@v6
 424
 425      - name: ccache
 426        uses: ggml-org/ccache-action@v1.2.16
 427        with:
 428          key: ubuntu-24-cmake-vulkan
 429          evict-old-files: 1d
 430          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 431
 432      - name: Dependencies
 433        id: depends
 434        run: |
 435          sudo add-apt-repository -y ppa:kisak/kisak-mesa
 436          sudo apt-get update -y
 437          sudo apt-get install -y build-essential mesa-vulkan-drivers libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libssl-dev
 438
 439      - name: Get latest Vulkan SDK version
 440        id: vulkan_sdk_version
 441        run: |
 442          echo "VULKAN_SDK_VERSION=$(curl https://vulkan.lunarg.com/sdk/latest/linux.txt)" >> "$GITHUB_ENV"
 443
 444      - name: Use Vulkan SDK Cache
 445        uses: actions/cache@v5
 446        id: cache-sdk
 447        with:
 448          path: ./vulkan_sdk
 449          key: vulkan-sdk-${{ env.VULKAN_SDK_VERSION }}-${{ runner.os }}
 450
 451      - name: Setup Vulkan SDK
 452        if: steps.cache-sdk.outputs.cache-hit != 'true'
 453        uses: ./.github/actions/linux-setup-vulkan
 454        with:
 455          path: ./vulkan_sdk
 456          version: ${{ env.VULKAN_SDK_VERSION }}
 457
 458      - name: Build
 459        id: cmake_build
 460        run: |
 461          source ./vulkan_sdk/setup-env.sh
 462          cmake -B build \
 463            -DGGML_VULKAN=ON
 464          cmake --build build --config Release -j $(nproc)
 465
 466      - name: Test
 467        id: cmake_test
 468        run: |
 469          cd build
 470          export GGML_VK_VISIBLE_DEVICES=0
 471          export GGML_VK_DISABLE_F16=1
 472          # This is using llvmpipe and runs slower than other backends
 473          ctest -L main --verbose --timeout 4800
 474
 475  ubuntu-24-cmake-webgpu:
 476    runs-on: ubuntu-24.04
 477
 478    steps:
 479      - name: Clone
 480        id: checkout
 481        uses: actions/checkout@v6
 482
 483      - name: ccache
 484        uses: ggml-org/ccache-action@v1.2.16
 485        with:
 486          key: ubuntu-24-cmake-webgpu
 487          evict-old-files: 1d
 488          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 489
 490      - name: Dependencies
 491        id: depends
 492        run: |
 493          sudo add-apt-repository -y ppa:kisak/kisak-mesa
 494          sudo apt-get update -y
 495          sudo apt-get install -y build-essential mesa-vulkan-drivers libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libssl-dev
 496
 497      - name: Get latest Vulkan SDK version
 498        id: vulkan_sdk_version
 499        run: |
 500          echo "VULKAN_SDK_VERSION=$(curl https://vulkan.lunarg.com/sdk/latest/linux.txt)" >> "$GITHUB_ENV"
 501
 502      - name: Use Vulkan SDK Cache
 503        uses: actions/cache@v5
 504        id: cache-sdk
 505        with:
 506          path: ./vulkan_sdk
 507          key: vulkan-sdk-${{ env.VULKAN_SDK_VERSION }}-${{ runner.os }}
 508
 509      - name: Setup Vulkan SDK
 510        if: steps.cache-sdk.outputs.cache-hit != 'true'
 511        uses: ./.github/actions/linux-setup-vulkan
 512        with:
 513          path: ./vulkan_sdk
 514          version: ${{ env.VULKAN_SDK_VERSION }}
 515
 516      - name: Dawn Dependency
 517        id: dawn-depends
 518        run: |
 519          sudo apt-get install -y libxrandr-dev libxinerama-dev libxcursor-dev mesa-common-dev libx11-xcb-dev libxi-dev
 520          DAWN_VERSION="v2.0.0"
 521          DAWN_OWNER="reeselevine"
 522          DAWN_REPO="dawn"
 523          DAWN_ASSET_NAME="Dawn-5e9a4865b1635796ccc77dd30057f2b4002a1355-ubuntu-latest-Release"
 524          echo "Fetching release asset from https://github.com/${DAWN_OWNER}/${DAWN_REPO}/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.zip"
 525          curl -L -o artifact.zip \
 526            "https://github.com/${DAWN_OWNER}/${DAWN_REPO}/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.zip"
 527          mkdir dawn
 528          unzip artifact.zip
 529          tar -xvf ${DAWN_ASSET_NAME}.tar.gz -C dawn --strip-components=1
 530
 531      - name: Build
 532        id: cmake_build
 533        run: |
 534          export Dawn_DIR=dawn/lib64/cmake/Dawn
 535          cmake -B build \
 536            -DGGML_WEBGPU=ON
 537          cmake --build build --config Release -j $(nproc)
 538
 539      - name: Test
 540        id: cmake_test
 541        run: |
 542          cd build
 543          # This is using llvmpipe and runs slower than other backends
 544          ctest -L main --verbose --timeout 3600
 545
 546  ubuntu-24-wasm-webgpu:
 547    runs-on: ubuntu-24.04
 548
 549    steps:
 550      - name: Clone
 551        id: checkout
 552        uses: actions/checkout@v6
 553
 554      - name: ccache
 555        uses: ggml-org/ccache-action@v1.2.16
 556        with:
 557          key: ubuntu-latest-wasm-webgpu
 558          evict-old-files: 1d
 559          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 560
 561      - name: Install Emscripten
 562        run: |
 563          git clone https://github.com/emscripten-core/emsdk.git
 564          cd emsdk
 565          ./emsdk install latest
 566          ./emsdk activate latest
 567
 568      - name: Fetch emdawnwebgpu
 569        run: |
 570          DAWN_TAG="v20251027.212519"
 571          EMDAWN_PKG="emdawnwebgpu_pkg-${DAWN_TAG}.zip"
 572          echo "Downloading ${EMDAWN_PKG}"
 573          curl -L -o emdawn.zip \
 574            "https://github.com/google/dawn/releases/download/${DAWN_TAG}/${EMDAWN_PKG}"
 575          unzip emdawn.zip
 576
 577      - name: Build WASM WebGPU
 578        run: |
 579          source emsdk/emsdk_env.sh
 580          emcmake cmake -B build-wasm \
 581            -DGGML_WEBGPU=ON \
 582            -DLLAMA_OPENSSL=OFF \
 583            -DEMDAWNWEBGPU_DIR=emdawnwebgpu_pkg
 584
 585          cmake --build build-wasm --target test-backend-ops -j $(nproc)
 586
 587  ubuntu-22-cmake-hip:
 588    runs-on: ubuntu-22.04
 589    container: rocm/dev-ubuntu-22.04:6.1.2
 590
 591    steps:
 592      - name: Clone
 593        id: checkout
 594        uses: actions/checkout@v6
 595
 596      - name: Dependencies
 597        id: depends
 598        run: |
 599          sudo apt-get update
 600          sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev rocwmma-dev
 601
 602      - name: ccache
 603        uses: ggml-org/ccache-action@v1.2.16
 604        with:
 605          key: ubuntu-22-cmake-hip
 606          evict-old-files: 1d
 607          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 608
 609      - name: Build with native CMake HIP support
 610        id: cmake_build
 611        run: |
 612          cmake -B build -S . \
 613            -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \
 614            -DGGML_HIP_ROCWMMA_FATTN=ON \
 615            -DGGML_HIP=ON
 616          cmake --build build --config Release -j $(nproc)
 617
 618  ubuntu-22-cmake-musa:
 619    runs-on: ubuntu-22.04
 620    container: mthreads/musa:rc4.3.0-devel-ubuntu22.04-amd64
 621
 622    steps:
 623      - name: Clone
 624        id: checkout
 625        uses: actions/checkout@v6
 626
 627      - name: Dependencies
 628        id: depends
 629        run: |
 630          apt-get update
 631          apt-get install -y build-essential git cmake libssl-dev
 632
 633      - name: ccache
 634        uses: ggml-org/ccache-action@v1.2.16
 635        with:
 636          key: ubuntu-22-cmake-musa
 637          evict-old-files: 1d
 638          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 639
 640      - name: Build with native CMake MUSA support
 641        id: cmake_build
 642        run: |
 643          cmake -B build -S . \
 644            -DGGML_MUSA=ON
 645          cmake --build build --config Release -j $(nproc)
 646
 647  ubuntu-22-cmake-sycl:
 648    runs-on: ubuntu-22.04
 649
 650    continue-on-error: true
 651
 652    steps:
 653      - uses: actions/checkout@v6
 654
 655      - name: add oneAPI to apt
 656        shell: bash
 657        run: |
 658          cd /tmp
 659          wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
 660          sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
 661          rm GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
 662          sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
 663
 664      - name: install oneAPI dpcpp compiler
 665        shell: bash
 666        run: |
 667          sudo apt update
 668          sudo apt install intel-oneapi-compiler-dpcpp-cpp libssl-dev
 669
 670      - name: install oneAPI MKL library
 671        shell: bash
 672        run: |
 673          sudo apt install intel-oneapi-mkl-devel
 674
 675      - name: Clone
 676        id: checkout
 677        uses: actions/checkout@v6
 678
 679      - name: ccache
 680        uses: ggml-org/ccache-action@v1.2.16
 681        with:
 682          key: ubuntu-22-cmake-sycl
 683          evict-old-files: 1d
 684          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 685
 686      - name: Build
 687        id: cmake_build
 688        run: |
 689          source /opt/intel/oneapi/setvars.sh
 690          cmake -B build \
 691            -DGGML_SYCL=ON \
 692            -DCMAKE_C_COMPILER=icx \
 693            -DCMAKE_CXX_COMPILER=icpx
 694          cmake --build build --config Release -j $(nproc)
 695
 696  ubuntu-22-cmake-sycl-fp16:
 697    runs-on: ubuntu-22.04
 698
 699    continue-on-error: true
 700
 701    steps:
 702      - uses: actions/checkout@v6
 703
 704      - name: add oneAPI to apt
 705        shell: bash
 706        run: |
 707          cd /tmp
 708          wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
 709          sudo apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
 710          rm GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB
 711          sudo add-apt-repository "deb https://apt.repos.intel.com/oneapi all main"
 712
 713      - name: install oneAPI dpcpp compiler
 714        shell: bash
 715        run: |
 716          sudo apt update
 717          sudo apt install intel-oneapi-compiler-dpcpp-cpp libssl-dev
 718
 719      - name: install oneAPI MKL library
 720        shell: bash
 721        run: |
 722          sudo apt install intel-oneapi-mkl-devel
 723
 724      - name: Clone
 725        id: checkout
 726        uses: actions/checkout@v6
 727
 728      - name: ccache
 729        uses: ggml-org/ccache-action@v1.2.16
 730        with:
 731          key: ubuntu-22-cmake-sycl-fp16
 732          evict-old-files: 1d
 733          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 734
 735      - name: Build
 736        id: cmake_build
 737        run: |
 738          source /opt/intel/oneapi/setvars.sh
 739          cmake -B build \
 740            -DGGML_SYCL=ON \
 741            -DCMAKE_C_COMPILER=icx \
 742            -DCMAKE_CXX_COMPILER=icpx \
 743            -DGGML_SYCL_F16=ON
 744          cmake --build build --config Release -j $(nproc)
 745
 746  build-linux-cross:
 747    uses: ./.github/workflows/build-linux-cross.yml
 748
 749  build-cmake-pkg:
 750    uses: ./.github/workflows/build-cmake-pkg.yml
 751
 752  macOS-latest-cmake-ios:
 753    runs-on: macos-latest
 754
 755    steps:
 756      - name: Clone
 757        id: checkout
 758        uses: actions/checkout@v6
 759
 760      - name: ccache
 761        uses: ggml-org/ccache-action@v1.2.16
 762        with:
 763          key: macOS-latest-cmake-ios
 764          evict-old-files: 1d
 765          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 766
 767      - name: Build
 768        id: cmake_build
 769        run: |
 770          sysctl -a
 771          cmake -B build -G Xcode \
 772            -DGGML_METAL_USE_BF16=ON \
 773            -DGGML_METAL_EMBED_LIBRARY=ON \
 774            -DLLAMA_BUILD_COMMON=OFF \
 775            -DLLAMA_BUILD_EXAMPLES=OFF \
 776            -DLLAMA_BUILD_TOOLS=OFF \
 777            -DLLAMA_BUILD_TESTS=OFF \
 778            -DLLAMA_BUILD_SERVER=OFF \
 779            -DCMAKE_SYSTEM_NAME=iOS \
 780            -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
 781            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
 782          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
 783
 784  macOS-latest-cmake-tvos:
 785    runs-on: macos-latest
 786
 787    steps:
 788      - name: Clone
 789        id: checkout
 790        uses: actions/checkout@v6
 791
 792      - name: ccache
 793        uses: ggml-org/ccache-action@v1.2.16
 794        with:
 795          key: macOS-latest-cmake-tvos
 796          evict-old-files: 1d
 797          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 798
 799      - name: Build
 800        id: cmake_build
 801        run: |
 802          sysctl -a
 803          cmake -B build -G Xcode \
 804            -DGGML_METAL_USE_BF16=ON \
 805            -DGGML_METAL_EMBED_LIBRARY=ON \
 806            -DLLAMA_BUILD_COMMON=OFF \
 807            -DLLAMA_BUILD_EXAMPLES=OFF \
 808            -DLLAMA_BUILD_TOOLS=OFF \
 809            -DLLAMA_BUILD_TESTS=OFF \
 810            -DLLAMA_BUILD_SERVER=OFF \
 811            -DCMAKE_SYSTEM_NAME=tvOS \
 812            -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
 813            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
 814          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
 815
 816  macOS-latest-cmake-visionos:
 817    runs-on: macos-latest
 818
 819    steps:
 820      - name: Clone
 821        id: checkout
 822        uses: actions/checkout@v6
 823
 824      - name: Build
 825        id: cmake_build
 826        run: |
 827          sysctl -a
 828          cmake -B build -G Xcode \
 829            -DGGML_METAL_USE_BF16=ON \
 830            -DGGML_METAL_EMBED_LIBRARY=ON \
 831            -DLLAMA_BUILD_COMMON=OFF \
 832            -DLLAMA_BUILD_EXAMPLES=OFF \
 833            -DLLAMA_BUILD_TOOLS=OFF \
 834            -DLLAMA_BUILD_TESTS=OFF \
 835            -DLLAMA_BUILD_SERVER=OFF \
 836            -DCMAKE_SYSTEM_NAME=visionOS \
 837            -DCMAKE_OSX_DEPLOYMENT_TARGET=1.0 \
 838            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
 839          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
 840
 841  macOS-latest-swift:
 842    runs-on: macos-latest
 843    needs: ios-xcode-build
 844
 845    strategy:
 846      matrix:
 847        destination: ['generic/platform=macOS', 'generic/platform=iOS', 'generic/platform=tvOS']
 848
 849    steps:
 850      - name: Clone
 851        id: checkout
 852        uses: actions/checkout@v6
 853
 854      - name: ccache
 855        uses: ggml-org/ccache-action@v1.2.16
 856        with:
 857          key: macOS-latest-swift
 858          evict-old-files: 1d
 859          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 860
 861      - name: Download xcframework artifact
 862        uses: actions/download-artifact@v7
 863        with:
 864          name: llama-xcframework
 865          path: build-apple/llama.xcframework/
 866
 867      - name: Build llama.cpp with CMake
 868        id: cmake_build
 869        run: |
 870          sysctl -a
 871          cmake -B build -G Xcode \
 872            -DGGML_METAL_USE_BF16=ON \
 873            -DGGML_METAL_EMBED_LIBRARY=ON \
 874            -DLLAMA_OPENSSL=OFF \
 875            -DLLAMA_BUILD_EXAMPLES=OFF \
 876            -DLLAMA_BUILD_TOOLS=OFF \
 877            -DLLAMA_BUILD_TESTS=OFF \
 878            -DLLAMA_BUILD_SERVER=OFF \
 879            -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64"
 880          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
 881
 882  windows-msys2:
 883    runs-on: windows-2025
 884
 885    strategy:
 886      fail-fast: false
 887      matrix:
 888        include:
 889          - { sys: UCRT64,  env: ucrt-x86_64,  build: Release }
 890          - { sys: CLANG64, env: clang-x86_64, build: Release }
 891
 892    steps:
 893      - name: Clone
 894        uses: actions/checkout@v6
 895
 896      - name: ccache
 897        uses: ggml-org/ccache-action@v1.2.16
 898        with:
 899          key: windows-msys2
 900          variant: ccache
 901          evict-old-files: 1d
 902          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 903
 904      - name: Setup ${{ matrix.sys }}
 905        uses: msys2/setup-msys2@v2
 906        with:
 907          update: true
 908          msystem: ${{matrix.sys}}
 909          install: >-
 910            base-devel
 911            git
 912            mingw-w64-${{matrix.env}}-toolchain
 913            mingw-w64-${{matrix.env}}-cmake
 914            mingw-w64-${{matrix.env}}-openblas
 915
 916      - name: Build using CMake
 917        shell: msys2 {0}
 918        run: |
 919            cmake -B build
 920            cmake --build build --config ${{ matrix.build }} -j $(nproc)
 921
 922      - name: Clean after building using CMake
 923        shell: msys2 {0}
 924        run: |
 925            rm -rf build
 926
 927      - name: Build using CMake w/ OpenBLAS
 928        shell: msys2 {0}
 929        run: |
 930            cmake -B build -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
 931            cmake --build build --config ${{ matrix.build }} -j $(nproc)
 932
 933  windows-latest-cmake:
 934    runs-on: windows-2025
 935
 936    env:
 937      OPENBLAS_VERSION: 0.3.23
 938      SDE_VERSION: 9.33.0-2024-01-07
 939      VULKAN_VERSION: 1.4.313.2
 940
 941    strategy:
 942      matrix:
 943        include:
 944          - build: 'cpu-x64 (static)'
 945            arch: 'x64'
 946            defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF'
 947          - build: 'openblas-x64'
 948            arch: 'x64'
 949            defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_OPENMP=OFF -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
 950          - build: 'vulkan-x64'
 951            arch: 'x64'
 952            defines: '-DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_VULKAN=ON'
 953          - build: 'llvm-arm64'
 954            arch: 'arm64'
 955            defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON'
 956          - build: 'llvm-arm64-opencl-adreno'
 957            arch: 'arm64'
 958            defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" -DGGML_OPENCL=ON -DGGML_OPENCL_USE_ADRENO_KERNELS=ON'
 959
 960    steps:
 961      - name: Clone
 962        id: checkout
 963        uses: actions/checkout@v6
 964
 965      - name: ccache
 966        uses: ggml-org/ccache-action@v1.2.16
 967        with:
 968          key: windows-latest-cmake-${{ matrix.build }}
 969          variant: ccache
 970          evict-old-files: 1d
 971          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
 972
 973      - name: Download OpenBLAS
 974        id: get_openblas
 975        if: ${{ matrix.build == 'openblas-x64' }}
 976        run: |
 977          curl.exe -o $env:RUNNER_TEMP/openblas.zip -L "https://github.com/xianyi/OpenBLAS/releases/download/v${env:OPENBLAS_VERSION}/OpenBLAS-${env:OPENBLAS_VERSION}-x64.zip"
 978          curl.exe -o $env:RUNNER_TEMP/OpenBLAS.LICENSE.txt -L "https://github.com/xianyi/OpenBLAS/raw/v${env:OPENBLAS_VERSION}/LICENSE"
 979          mkdir $env:RUNNER_TEMP/openblas
 980          tar.exe -xvf $env:RUNNER_TEMP/openblas.zip -C $env:RUNNER_TEMP/openblas
 981          $vcdir = $(vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath)
 982          $msvc = $(join-path $vcdir $('VC\Tools\MSVC\'+$(gc -raw $(join-path $vcdir 'VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt')).Trim()))
 983          $lib =  $(join-path $msvc 'bin\Hostx64\x64\lib.exe')
 984          & $lib /machine:x64 "/def:${env:RUNNER_TEMP}/openblas/lib/libopenblas.def" "/out:${env:RUNNER_TEMP}/openblas/lib/openblas.lib" /name:openblas.dll
 985
 986      - name: Install Vulkan SDK
 987        id: get_vulkan
 988        if: ${{ matrix.build == 'vulkan-x64' }}
 989        run: |
 990          curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"
 991          & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install
 992          Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
 993          Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"
 994
 995      - name: Install Ninja
 996        id: install_ninja
 997        run: |
 998          choco install ninja
 999
1000      - name: Install OpenCL Headers and Libs
1001        id: install_opencl
1002        if: ${{ matrix.build == 'llvm-arm64-opencl-adreno' }}
1003        run: |
1004          git clone https://github.com/KhronosGroup/OpenCL-Headers
1005          cd OpenCL-Headers
1006          cmake -B build `
1007            -DBUILD_TESTING=OFF `
1008            -DOPENCL_HEADERS_BUILD_TESTING=OFF `
1009            -DOPENCL_HEADERS_BUILD_CXX_TESTS=OFF `
1010            -DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release"
1011          cmake --build build --target install
1012          git clone https://github.com/KhronosGroup/OpenCL-ICD-Loader
1013          cd OpenCL-ICD-Loader
1014          cmake -B build-arm64-release `
1015            -A arm64 `
1016            -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" `
1017            -DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release"
1018          cmake --build build-arm64-release --target install --config release
1019
1020      - name: Build
1021        id: cmake_build
1022        run: |
1023          cmake -S . -B build ${{ matrix.defines }} `
1024            -DLLAMA_BUILD_BORINGSSL=ON
1025          cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}
1026
1027      - name: Add libopenblas.dll
1028        id: add_libopenblas_dll
1029        if: ${{ matrix.build == 'openblas-x64' }}
1030        run: |
1031          cp $env:RUNNER_TEMP/openblas/bin/libopenblas.dll ./build/bin/Release/openblas.dll
1032          cp $env:RUNNER_TEMP/OpenBLAS.LICENSE.txt ./build/bin/Release/OpenBLAS-${env:OPENBLAS_VERSION}.txt
1033
1034      - name: Test
1035        id: cmake_test
1036        if: ${{ matrix.arch == 'x64' }}
1037        run: |
1038          cd build
1039          ctest -L main -C Release --verbose --timeout 900
1040
1041      # TODO: disabled for now, consider adding tests for all CPU variants instead
1042      # - name: Test (Intel SDE)
1043      #   id: cmake_test_sde
1044      #   if: ${{ matrix.build == 'avx512-x64' && env.HAS_AVX512F == '0' }} # use Intel SDE for AVX-512 emulation
1045      #   run: |
1046      #     curl.exe -o $env:RUNNER_TEMP/sde.tar.xz -L "https://downloadmirror.intel.com/813591/sde-external-${env:SDE_VERSION}-win.tar.xz"
1047      #     # for some weird reason windows tar doesn't like sde tar.xz
1048      #     7z x "-o${env:RUNNER_TEMP}" $env:RUNNER_TEMP/sde.tar.xz
1049      #     7z x "-o${env:RUNNER_TEMP}" $env:RUNNER_TEMP/sde.tar
1050      #     $sde = $(join-path $env:RUNNER_TEMP sde-external-${env:SDE_VERSION}-win/sde.exe)
1051      #     cd build
1052      #     $env:LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR = 1
1053      #     & $sde -future -- ctest -L main -C Release --verbose --timeout 900
1054
1055  ubuntu-latest-cmake-cuda:
1056    runs-on: ubuntu-latest
1057    container: nvidia/cuda:12.6.2-devel-ubuntu24.04
1058
1059    steps:
1060        - name: Clone
1061          id: checkout
1062          uses: actions/checkout@v6
1063
1064        - name: Install dependencies
1065          env:
1066            DEBIAN_FRONTEND: noninteractive
1067          run: |
1068              apt update
1069              apt install -y cmake build-essential ninja-build libgomp1 git libssl-dev
1070
1071        - name: ccache
1072          uses: ggml-org/ccache-action@v1.2.16
1073          with:
1074            key: ubuntu-latest-cmake-cuda
1075            evict-old-files: 1d
1076            save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
1077
1078        - name: Build with CMake
1079          # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project
1080          run: |
1081            cmake -S . -B build -G Ninja \
1082              -DLLAMA_FATAL_WARNINGS=ON \
1083              -DCMAKE_BUILD_TYPE=Release \
1084              -DCMAKE_CUDA_ARCHITECTURES=89-real \
1085              -DCMAKE_EXE_LINKER_FLAGS=-Wl,--allow-shlib-undefined \
1086              -DGGML_NATIVE=OFF \
1087              -DGGML_CUDA=ON \
1088              -DGGML_CUDA_CUB_3DOT2=ON
1089            cmake --build build
1090
1091  windows-2022-cmake-cuda:
1092    runs-on: windows-2022
1093
1094    strategy:
1095      matrix:
1096        cuda: ['12.4']
1097
1098    steps:
1099      - name: Clone
1100        id: checkout
1101        uses: actions/checkout@v6
1102
1103      - name: Install ccache
1104        uses: ggml-org/ccache-action@v1.2.16
1105        with:
1106          key: windows-cuda-${{ matrix.cuda }}
1107          variant: ccache
1108          evict-old-files: 1d
1109          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
1110
1111      - name: Install Cuda Toolkit
1112        uses: ./.github/actions/windows-setup-cuda
1113        with:
1114          cuda_version: ${{ matrix.cuda }}
1115
1116      - name: Install Ninja
1117        id: install_ninja
1118        run: |
1119          choco install ninja
1120
1121      - name: Build
1122        id: cmake_build
1123        shell: cmd
1124        # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project
1125        run: |
1126          call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
1127          cmake -S . -B build -G "Ninja Multi-Config" ^
1128            -DLLAMA_BUILD_SERVER=ON ^
1129            -DLLAMA_BUILD_BORINGSSL=ON ^
1130            -DGGML_NATIVE=OFF ^
1131            -DGGML_BACKEND_DL=ON ^
1132            -DGGML_CPU_ALL_VARIANTS=ON ^
1133            -DGGML_CUDA=ON ^
1134            -DGGML_RPC=ON ^
1135            -DGGML_CUDA_CUB_3DOT2=ON
1136          set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1
1137          cmake --build build --config Release -j %NINJA_JOBS% -t ggml
1138          cmake --build build --config Release
1139
1140  windows-latest-cmake-sycl:
1141    runs-on: windows-2022
1142
1143    defaults:
1144      run:
1145        shell: bash
1146
1147    env:
1148      WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/24751ead-ddc5-4479-b9e6-f9fe2ff8b9f2/intel-deep-learning-essentials-2025.2.1.25_offline.exe
1149      WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel
1150      ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
1151    steps:
1152      - name: Clone
1153        id: checkout
1154        uses: actions/checkout@v6
1155
1156      - name: ccache
1157        uses: ggml-org/ccache-action@v1.2.16
1158        with:
1159          key: windows-latest-cmake-sycl
1160          variant: ccache
1161          evict-old-files: 1d
1162          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
1163
1164      - name: Install
1165        run:  |
1166          scripts/install-oneapi.bat $WINDOWS_BASEKIT_URL $WINDOWS_DPCPP_MKL
1167
1168      # TODO: add ssl support ; we will also need to modify win-build-sycl.bat to accept user-specified args
1169
1170      - name: Build
1171        id: cmake_build
1172        run:  examples/sycl/win-build-sycl.bat
1173
1174  windows-latest-cmake-hip:
1175    runs-on: windows-2022
1176
1177    env:
1178      # The ROCm version must correspond to the version used in the HIP SDK.
1179      ROCM_VERSION: "6.4.2"
1180      # Make sure this is in sync with build-cache.yml
1181      HIPSDK_INSTALLER_VERSION: "25.Q3"
1182
1183    steps:
1184      - name: Clone
1185        id: checkout
1186        uses: actions/checkout@v6
1187
1188      - name: Grab rocWMMA package
1189        id: grab_rocwmma
1190        run: |
1191          curl -o rocwmma.deb "https://repo.radeon.com/rocm/apt/${{ env.ROCM_VERSION }}/pool/main/r/rocwmma-dev/rocwmma-dev_1.7.0.60402-120~24.04_amd64.deb"
1192          7z x rocwmma.deb
1193          7z x data.tar
1194
1195      - name: Use ROCm Installation Cache
1196        uses: actions/cache@v5
1197        id: cache-rocm
1198        with:
1199          path: C:\Program Files\AMD\ROCm
1200          key: rocm-${{ env.HIPSDK_INSTALLER_VERSION }}-${{ runner.os }}
1201
1202      - name: Setup ROCm
1203        if: steps.cache-rocm.outputs.cache-hit != 'true'
1204        uses: ./.github/actions/windows-setup-rocm
1205        with:
1206          version: ${{ env.HIPSDK_INSTALLER_VERSION }}
1207
1208      - name: Verify ROCm
1209        id: verify
1210        run: |
1211          # Find and test ROCm installation
1212          $clangPath = Get-ChildItem 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | Select-Object -First 1
1213          if (-not $clangPath) {
1214            Write-Error "ROCm installation not found"
1215            exit 1
1216          }
1217          & $clangPath.FullName --version
1218
1219      - name: Install ccache
1220        uses: ggml-org/ccache-action@v1.2.16
1221        with:
1222          key: ${{ github.job }}
1223          evict-old-files: 1d
1224          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
1225
1226      - name: Build
1227        id: cmake_build
1228        run: |
1229          $env:HIP_PATH=$(Resolve-Path 'C:\Program Files\AMD\ROCm\*\bin\clang.exe' | split-path | split-path)
1230          $env:CMAKE_PREFIX_PATH="${env:HIP_PATH}"
1231          cmake -G "Unix Makefiles" -B build -S . `
1232            -DCMAKE_C_COMPILER="${env:HIP_PATH}\bin\clang.exe" `
1233            -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\bin\clang++.exe" `
1234            -DCMAKE_CXX_FLAGS="-I$($PWD.Path.Replace('\', '/'))/opt/rocm-${{ env.ROCM_VERSION }}/include/" `
1235            -DCMAKE_BUILD_TYPE=Release `
1236            -DLLAMA_BUILD_BORINGSSL=ON `
1237            -DROCM_DIR="${env:HIP_PATH}" `
1238            -DGGML_HIP=ON `
1239            -DGGML_HIP_ROCWMMA_FATTN=ON `
1240            -DGGML_RPC=ON
1241          cmake --build build -j ${env:NUMBER_OF_PROCESSORS}
1242
1243  ios-xcode-build:
1244    runs-on: macos-latest
1245
1246    steps:
1247      - name: Checkout code
1248        uses: actions/checkout@v6
1249
1250      - name: Setup Xcode
1251        uses: maxim-lobanov/setup-xcode@v1
1252        with:
1253          xcode-version: latest-stable
1254
1255      - name: Build
1256        id: cmake_build
1257        run: |
1258          sysctl -a
1259          cmake -B build -G Xcode \
1260            -DGGML_METAL_USE_BF16=ON \
1261            -DGGML_METAL_EMBED_LIBRARY=ON \
1262            -DLLAMA_OPENSSL=OFF \
1263            -DLLAMA_BUILD_EXAMPLES=OFF \
1264            -DLLAMA_BUILD_TOOLS=OFF \
1265            -DLLAMA_BUILD_TESTS=OFF \
1266            -DLLAMA_BUILD_SERVER=OFF \
1267            -DCMAKE_SYSTEM_NAME=iOS \
1268            -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
1269            -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
1270          cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
1271
1272      - name: xcodebuild for swift package
1273        id: xcodebuild
1274        run: |
1275          ./build-xcframework.sh
1276
1277      - name: Upload xcframework artifact
1278        uses: actions/upload-artifact@v6
1279        with:
1280          name: llama-xcframework
1281          path: build-apple/llama.xcframework/
1282          retention-days: 1
1283
1284      - name: Build Xcode project
1285        run: |
1286          xcodebuild -downloadPlatform iOS
1287          xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build
1288
1289  android-build:
1290    runs-on: ubuntu-latest
1291
1292    steps:
1293      - name: Clone
1294        uses: actions/checkout@v6
1295
1296      # Disabled due to size (400MB) and always 0 cache hits
1297      # - name: ccache
1298      #   uses: ggml-org/ccache-action@v1.2.16
1299      #   with:
1300      #     key: android-build
1301      #     evict-old-files: 1d
1302
1303      - name: Set up JDK
1304        uses: actions/setup-java@v5
1305        with:
1306          java-version: 17
1307          distribution: zulu
1308
1309      - name: Setup Android SDK
1310        uses: android-actions/setup-android@v3
1311        with:
1312          log-accepted-android-sdk-licenses: false
1313
1314      - name: Build
1315        run: |
1316          cd examples/llama.android
1317          ./gradlew build --no-daemon
1318
1319  android-ndk-build:
1320    runs-on: ubuntu-latest
1321
1322    env:
1323      OPENCL_VERSION: 2025.07.22
1324
1325    strategy:
1326      matrix:
1327        include:
1328          - build: 'arm64-cpu'
1329            defines: '-D ANDROID_ABI=arm64-v8a -D ANDROID_PLATFORM=android-31 -D CMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake -D GGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8.5-a+fp16+i8mm -G Ninja -D LLAMA_OPENSSL=OFF -D GGML_OPENMP=OFF'
1330          - build: 'arm64-snapdragon'
1331            defines: '--preset arm64-android-snapdragon-release'
1332
1333    steps:
1334      - name: Clone
1335        id: checkout
1336        uses: actions/checkout@v6
1337
1338      - name: Install OpenCL Headers and Libs
1339        id: install_opencl
1340        if: ${{ matrix.build == 'arm64-snapdragon' }}
1341        run: |
1342          mkdir opencl
1343          curl -L -o opencl/clhpp.tar.gz      https://github.com/KhronosGroup/OpenCL-CLHPP/archive/refs/tags/v${OPENCL_VERSION}.tar.gz
1344          curl -L -o opencl/headers.tar.gz    https://github.com/KhronosGroup/OpenCL-Headers/archive/refs/tags/v${OPENCL_VERSION}.tar.gz
1345          curl -L -o opencl/icd-loader.tar.gz https://github.com/KhronosGroup/OpenCL-ICD-Loader/archive/refs/tags/v${OPENCL_VERSION}.tar.gz
1346          tar -xaf opencl/headers.tar.gz    -C opencl
1347          tar -xaf opencl/clhpp.tar.gz      -C opencl
1348          tar -xaf opencl/icd-loader.tar.gz -C opencl
1349          sudo cp -r opencl/OpenCL-Headers-${OPENCL_VERSION}/CL         ${ANDROID_NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include
1350          sudo cp -r opencl/OpenCL-CLHPP-${OPENCL_VERSION}/include/CL/* ${ANDROID_NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include/CL
1351          cd opencl/OpenCL-ICD-Loader-${OPENCL_VERSION}
1352          cmake -B build -G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake -DOPENCL_ICD_LOADER_HEADERS_DIR=${ANDROID_NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=31 -DANDROID_STL=c++_shared
1353          cmake --build build
1354          sudo cp build/libOpenCL.so ${ANDROID_NDK_ROOT}/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/aarch64-linux-android
1355          rm -rf opencl
1356
1357      - name: Install Hexagon SDK
1358        id: install_hexsdk
1359        if: ${{ matrix.build == 'arm64-snapdragon' }}
1360        env:
1361          HEXSDK_VER: 6.4.0.2
1362          HEXTLS_VER: 19.0.04
1363        run: |
1364          curl -L -o hex-sdk.tar.gz https://github.com/snapdragon-toolchain/hexagon-sdk/releases/download/v$HEXSDK_VER/hexagon-sdk-v$HEXSDK_VER-amd64-lnx.tar.xz
1365          mkdir hex-sdk
1366          tar -xaf hex-sdk.tar.gz -C hex-sdk
1367          ls -l hex-sdk
1368          sudo mv hex-sdk /opt/hexagon
1369          echo "HEXAGON_SDK_ROOT=/opt/hexagon/$HEXSDK_VER"                                     >> "$GITHUB_ENV"
1370          echo "HEXAGON_TOOLS_ROOT=/opt/hexagon/$HEXSDK_VER/tools/HEXAGON_Tools/$HEXTLS_VER"   >> "$GITHUB_ENV"
1371          echo "DEFAULT_HLOS_ARCH=64"                                                          >> "$GITHUB_ENV"
1372          echo "DEFAULT_TOOLS_VARIANT=toolv19"                                                 >> "$GITHUB_ENV"
1373          echo "DEFAULT_NO_QURT_INC=0"                                                         >> "$GITHUB_ENV"
1374          echo "DEFAULT_DSP_ARCH=v73"                                                          >> "$GITHUB_ENV"
1375
1376      - name: Update CMake presets
1377        id: update_presets
1378        if: ${{ matrix.build == 'arm64-snapdragon' }}
1379        run: |
1380          cp docs/backend/snapdragon/CMakeUserPresets.json .
1381
1382      - name: Build
1383        id: ndk_build
1384        run: |
1385          cmake ${{ matrix.defines }} -B build
1386          cmake --build build
1387          cmake --install build --prefix pkg-adb/llama.cpp
1388
1389      - name: Test
1390        id: cmake_test
1391        run: |
1392          echo "FIXME: test on devices"
1393
1394  openEuler-latest-cmake-cann:
1395    defaults:
1396      run:
1397        shell: bash -el {0}
1398    strategy:
1399      matrix:
1400        arch: [x86, aarch64]
1401        chip_type: ['910b', '310p']
1402        build: ['Release']
1403        use_acl_graph: ['on', 'off']
1404        exclude:
1405          # 310P does not support USE_ACL_GRAPH=on
1406          - chip_type: '310p'
1407            use_acl_graph: 'on'
1408    runs-on: ${{ matrix.arch == 'aarch64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
1409    steps:
1410      - name: Checkout
1411        uses: actions/checkout@v6
1412        with:
1413          fetch-depth: 0
1414
1415      - name: Free up disk space
1416        uses: ggml-org/free-disk-space@v1.3.1
1417        with:
1418          tool-cache: true
1419
1420      - name: Set container image
1421        id: cann-image
1422        run: |
1423          image="ascendai/cann:${{ matrix.chip_type == '910b' &&  '8.3.rc2-910b-openeuler24.03-py3.11' || '8.3.rc2-310p-openeuler24.03-py3.11' }}"
1424          echo "image=${image}" >> "${GITHUB_OUTPUT}"
1425
1426      - name: Pull container image
1427        run: docker pull "${{ steps.cann-image.outputs.image }}"
1428
1429      - name: Build
1430        env:
1431          BUILD_TYPE: ${{ matrix.build }}
1432          SOC_TYPE: ascend${{ matrix.chip_type }}
1433          USE_ACL_GRAPH: ${{ matrix.use_acl_graph }}
1434        run: |
1435          HOST_UID=$(id -u)
1436          HOST_GID=$(id -g)
1437
1438          docker run --rm \
1439            -v "${PWD}:/workspace" \
1440            -w /workspace \
1441            -e SOC_TYPE=${SOC_TYPE} \
1442            -e BUILD_TYPE=${BUILD_TYPE} \
1443            -e USE_ACL_GRAPH=${USE_ACL_GRAPH} \
1444            "${{ steps.cann-image.outputs.image }}" \
1445            bash -lc '
1446              set -e
1447              yum install -y --setopt=install_weak_deps=False --setopt=tsflags=nodocs git gcc gcc-c++ make cmake openssl-devel
1448              yum clean all && rm -rf /var/cache/yum
1449              git config --global --add safe.directory "/workspace"
1450              export LD_LIBRARY_PATH=${ASCEND_TOOLKIT_HOME}/lib64:${ASCEND_TOOLKIT_HOME}/$(uname -m)-linux/devlib/:${LD_LIBRARY_PATH}
1451              cmake -S . -B build \
1452                  -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
1453                  -DGGML_CANN=on \
1454                  -DSOC_TYPE=${SOC_TYPE} \
1455                  -DUSE_ACL_GRAPH=${USE_ACL_GRAPH}
1456              cmake --build build -j $(nproc)
1457
1458              chown -R '"${HOST_UID}"':'"${HOST_GID}"' /workspace/build
1459            '
1460
1461# TODO: simplify the following workflows using a matrix
1462# TODO: run lighter CI on PRs and the full CI only on master (if needed)
1463  ggml-ci-x64-cpu-low-perf:
1464    runs-on: ubuntu-22.04
1465
1466    steps:
1467      - name: Clone
1468        id: checkout
1469        uses: actions/checkout@v6
1470
1471      - name: ccache
1472        uses: ggml-org/ccache-action@v1.2.16
1473        with:
1474          key: ggml-ci-x64-cpu-low-perf
1475          evict-old-files: 1d
1476          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
1477
1478      - name: Dependencies
1479        id: depends
1480        run: |
1481          sudo apt-get update
1482          sudo apt-get install build-essential
1483
1484      - name: Test
1485        id: ggml-ci
1486        run: |
1487          LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
1488
1489  ggml-ci-arm64-cpu-low-perf:
1490    runs-on: ubuntu-22.04-arm
1491
1492    steps:
1493      - name: Clone
1494        id: checkout
1495        uses: actions/checkout@v6
1496
1497      - name: ccache
1498        uses: ggml-org/ccache-action@v1.2.16
1499        with:
1500          key: ggml-ci-arm64-cpu-low-perf
1501          evict-old-files: 1d
1502          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
1503
1504      - name: Dependencies
1505        id: depends
1506        run: |
1507          sudo apt-get update
1508          sudo apt-get install build-essential
1509
1510      - name: Test
1511        id: ggml-ci
1512        run: |
1513          LLAMA_ARG_THREADS=$(nproc) GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
1514
1515  ggml-ci-x64-cpu-high-perf:
1516    runs-on: ubuntu-22.04
1517
1518    steps:
1519      - name: Clone
1520        id: checkout
1521        uses: actions/checkout@v6
1522
1523      - name: ccache
1524        uses: ggml-org/ccache-action@v1.2.16
1525        with:
1526          key: ggml-ci-x64-cpu-high-perf
1527          evict-old-files: 1d
1528          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
1529
1530      - name: Dependencies
1531        id: depends
1532        run: |
1533          sudo apt-get update
1534          sudo apt-get install build-essential
1535
1536      - name: Test
1537        id: ggml-ci
1538        run: |
1539          LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
1540
1541  ggml-ci-arm64-cpu-high-perf:
1542    runs-on: ubuntu-22.04-arm
1543
1544    steps:
1545      - name: Clone
1546        id: checkout
1547        uses: actions/checkout@v6
1548
1549      - name: ccache
1550        uses: ggml-org/ccache-action@v1.2.16
1551        with:
1552          key: ggml-ci-arm64-cpu-high-perf
1553          evict-old-files: 1d
1554          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
1555
1556      - name: Dependencies
1557        id: depends
1558        run: |
1559          sudo apt-get update
1560          sudo apt-get install build-essential
1561
1562      - name: Test
1563        id: ggml-ci
1564        run: |
1565          LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_NO_SVE=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
1566
1567  ggml-ci-arm64-cpu-high-perf-sve:
1568    runs-on: ubuntu-22.04-arm
1569
1570    steps:
1571      - name: Clone
1572        id: checkout
1573        uses: actions/checkout@v6
1574
1575      - name: ccache
1576        uses: ggml-org/ccache-action@v1.2.16
1577        with:
1578          key: ggml-ci-arm64-cpu-high-perf-sve
1579          evict-old-files: 1d
1580          save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
1581
1582      - name: Dependencies
1583        id: depends
1584        run: |
1585          sudo apt-get update
1586          sudo apt-get install build-essential
1587
1588      - name: Test
1589        id: ggml-ci
1590        run: |
1591          LLAMA_ARG_THREADS=$(nproc) GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
1592
1593  ggml-ci-x64-nvidia-cuda:
1594    runs-on: [self-hosted, Linux, X64, NVIDIA]
1595
1596    steps:
1597      - name: Clone
1598        id: checkout
1599        uses: actions/checkout@v6
1600
1601      - name: Test
1602        id: ggml-ci
1603        run: |
1604          nvidia-smi
1605          GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp
1606
1607  ggml-ci-x64-nvidia-vulkan-cm:
1608    runs-on: [self-hosted, Linux, X64, NVIDIA]
1609
1610    steps:
1611      - name: Clone
1612        id: checkout
1613        uses: actions/checkout@v6
1614
1615      - name: Test
1616        id: ggml-ci
1617        run: |
1618          vulkaninfo --summary
1619          GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp
1620
1621  ggml-ci-x64-nvidia-vulkan-cm2:
1622    runs-on: [self-hosted, Linux, X64, NVIDIA, COOPMAT2]
1623
1624    steps:
1625      - name: Clone
1626        id: checkout
1627        uses: actions/checkout@v6
1628
1629      - name: Test
1630        id: ggml-ci
1631        run: |
1632          vulkaninfo --summary
1633          GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp
1634
1635  ggml-ci-x64-cpu-amx:
1636    runs-on: [self-hosted, Linux, X64, CPU, AMX]
1637
1638    steps:
1639      - name: Clone
1640        id: checkout
1641        uses: actions/checkout@v6
1642
1643      - name: Test
1644        id: ggml-ci
1645        run: |
1646          bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp
1647
1648  # ggml-ci-x64-amd-vulkan:
1649  #   runs-on: [self-hosted, Linux, X64, AMD]
1650
1651  #   steps:
1652  #     - name: Clone
1653  #       id: checkout
1654  #       uses: actions/checkout@v6
1655
1656  #     - name: Test
1657  #       id: ggml-ci
1658  #       run: |
1659  #         vulkaninfo --summary
1660  #         GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp
1661
1662  # ggml-ci-x64-amd-rocm:
1663  #   runs-on: [self-hosted, Linux, X64, AMD]
1664
1665  #   steps:
1666  #     - name: Clone
1667  #       id: checkout
1668  #       uses: actions/checkout@v6
1669
1670  #     - name: Test
1671  #       id: ggml-ci
1672  #       run: |
1673  #         amd-smi static
1674  #         GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS="gfx1101" bash ./ci/run.sh ~/results/llama.cpp /mnt/llama.cpp
1675
1676  ggml-ci-mac-metal:
1677    runs-on: [self-hosted, macOS, ARM64]
1678
1679    steps:
1680      - name: Clone
1681        id: checkout
1682        uses: actions/checkout@v6
1683
1684      - name: Test
1685        id: ggml-ci
1686        run: |
1687          GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
1688
1689  ggml-ci-mac-webgpu:
1690    runs-on: [self-hosted, macOS, ARM64]
1691
1692    steps:
1693      - name: Clone
1694        id: checkout
1695        uses: actions/checkout@v6
1696
1697      - name: Dawn Dependency
1698        id: dawn-depends
1699        run: |
1700          DAWN_VERSION="v2.0.0"
1701          DAWN_OWNER="reeselevine"
1702          DAWN_REPO="dawn"
1703          DAWN_ASSET_NAME="Dawn-5e9a4865b1635796ccc77dd30057f2b4002a1355-macos-latest-Release"
1704          echo "Fetching release asset from https://github.com/${DAWN_OWNER}/${DAWN_REPO}/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.zip"
1705          curl -L -o artifact.zip \
1706            "https://github.com/${DAWN_OWNER}/${DAWN_REPO}/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.zip"
1707          mkdir dawn
1708          unzip artifact.zip
1709          tar -xvf ${DAWN_ASSET_NAME}.tar.gz -C dawn --strip-components=1
1710
1711      - name: Test
1712        id: ggml-ci
1713        run: |
1714          GG_BUILD_WEBGPU=1 GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
1715            bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
1716
1717  ggml-ci-mac-vulkan:
1718    runs-on: [self-hosted, macOS, ARM64]
1719
1720    steps:
1721      - name: Clone
1722        id: checkout
1723        uses: actions/checkout@v6
1724
1725      - name: Test
1726        id: ggml-ci
1727        run: |
1728          vulkaninfo --summary
1729          GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
1730
1731  ggml-ci-arm64-cpu-kleidiai:
1732     runs-on: ubuntu-22.04-arm
1733
1734     steps:
1735       - name: Clone
1736         id: checkout
1737         uses: actions/checkout@v6
1738
1739       - name: ccache
1740         uses: ggml-org/ccache-action@v1.2.16
1741         with:
1742           key: ggml-ci-arm64-cpu-kleidiai
1743           evict-old-files: 1d
1744           save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
1745
1746       - name: Dependencies
1747         id: depends
1748         run: |
1749           sudo apt-get update
1750           sudo apt-get install -y build-essential
1751
1752       - name: Test
1753         id: ggml-ci
1754         run: |
1755           GG_BUILD_KLEIDIAI=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ./tmp/results ./tmp/mnt
1756
1757  ubuntu-cpu-cmake-riscv64-native:
1758    runs-on: RISCV64
1759
1760    steps:
1761      - name: Install dependencies
1762        run: |
1763          sudo apt-get update
1764
1765          # Install necessary packages
1766          sudo apt-get install -y libatomic1 libtsan2 gcc-14 g++-14 rustup cmake build-essential libssl-dev wget ccache git-lfs
1767
1768          # Set gcc-14 and g++-14 as the default compilers
1769          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100
1770          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100
1771          sudo ln -sf /usr/bin/gcc-14 /usr/bin/gcc
1772          sudo ln -sf /usr/bin/g++-14 /usr/bin/g++
1773
1774          # Install Rust stable version
1775          rustup install stable
1776          rustup default stable
1777
1778          git lfs install
1779
1780      - name: Clone
1781        id: checkout
1782        uses: actions/checkout@v6
1783
1784      - name: Check environment
1785        run: |
1786          uname -a
1787          gcc --version
1788          g++ --version
1789          ldd --version
1790          cmake --version
1791          rustc --version
1792
1793      - name: Setup ccache
1794        run: |
1795          # Set unique cache directory for this job
1796          export CCACHE_DIR="$HOME/.ccache/cpu-cmake-rv64-native"
1797          mkdir -p "$CCACHE_DIR"
1798
1799          # Configure ccache for optimal performance
1800          ccache --set-config=max_size=5G
1801          ccache --set-config=compression=true
1802          ccache --set-config=compression_level=6
1803          ccache --set-config=cache_dir="$CCACHE_DIR"
1804
1805          # Enable more aggressive caching
1806          ccache --set-config=sloppiness=file_macro,time_macros,include_file_mtime,include_file_ctime
1807          ccache --set-config=hash_dir=false
1808
1809          # Export for subsequent steps
1810          echo "CCACHE_DIR=$CCACHE_DIR" >> $GITHUB_ENV
1811          echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
1812
1813      - name: Build
1814        id: cmake_build
1815        run: |
1816          cmake -B build \
1817            -DCMAKE_BUILD_TYPE=Release \
1818            -DGGML_OPENMP=OFF \
1819            -DLLAMA_BUILD_EXAMPLES=ON \
1820            -DLLAMA_BUILD_TOOLS=ON \
1821            -DLLAMA_BUILD_TESTS=ON \
1822            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
1823            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
1824            -DGGML_RPC=ON \
1825            -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
1826            -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14
1827
1828          cmake --build build --config Release -j $(nproc)
1829
1830      - name: Test
1831        id: cmake_test
1832        run: |
1833          cd build
1834          ctest -L main --verbose --timeout 900
1835
1836      - name: Test llama2c conversion
1837        id: llama2c_test
1838        run: |
1839          cd build
1840          echo "Fetch tokenizer"
1841          wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/tok512.bin
1842          echo "Fetch llama2c model"
1843          wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin
1844          ./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf
1845          ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
1846
1847  ubuntu-cmake-sanitizer-riscv64-native:
1848    runs-on: RISCV64
1849
1850    continue-on-error: true
1851
1852    strategy:
1853      matrix:
1854        sanitizer: [ADDRESS, THREAD, UNDEFINED]
1855        build_type: [Debug]
1856
1857    steps:
1858      - name: Install dependencies
1859        run: |
1860          sudo apt-get update
1861
1862          # Install necessary packages
1863          sudo apt-get install -y libatomic1 libtsan2 gcc-14 g++-14 rustup cmake build-essential wget ccache git-lfs
1864
1865          # Set gcc-14 and g++-14 as the default compilers
1866          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100
1867          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100
1868          sudo ln -sf /usr/bin/gcc-14 /usr/bin/gcc
1869          sudo ln -sf /usr/bin/g++-14 /usr/bin/g++
1870
1871          # Install Rust stable version
1872          rustup install stable
1873          rustup default stable
1874
1875          git lfs install
1876
1877      - name: GCC version check
1878        run: |
1879          gcc --version
1880          g++ --version
1881
1882      - name: Clone
1883        id: checkout
1884        uses: actions/checkout@v6
1885
1886      - name: Setup ccache
1887        run: |
1888          # Unique cache directory per matrix combination
1889          export CCACHE_DIR="$HOME/.ccache/sanitizer-${{ matrix.sanitizer }}-${{ matrix.build_type }}"
1890          mkdir -p "$CCACHE_DIR"
1891
1892          # Configure ccache
1893          ccache --set-config=max_size=5G
1894          ccache --set-config=compression=true
1895          ccache --set-config=compression_level=6
1896          ccache --set-config=cache_dir="$CCACHE_DIR"
1897          ccache --set-config=sloppiness=file_macro,time_macros,include_file_mtime,include_file_ctime
1898          ccache --set-config=hash_dir=false
1899
1900          # Export for subsequent steps
1901          echo "CCACHE_DIR=$CCACHE_DIR" >> $GITHUB_ENV
1902          echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
1903
1904      - name: Build
1905        id: cmake_build
1906        if: ${{ matrix.sanitizer != 'THREAD' }}
1907        run: |
1908          cmake -B build \
1909            -DLLAMA_OPENSSL=OFF \
1910            -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
1911            -DGGML_OPENMP=ON \
1912            -DLLAMA_BUILD_EXAMPLES=ON \
1913            -DLLAMA_BUILD_TOOLS=ON \
1914            -DLLAMA_BUILD_TESTS=OFF \
1915            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
1916            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
1917            -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
1918            -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
1919            -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14
1920
1921          cmake --build build --config ${{ matrix.build_type }} -j $(nproc)
1922
1923      - name: Build (no OpenMP)
1924        id: cmake_build_no_openmp
1925        if: ${{ matrix.sanitizer == 'THREAD' }}
1926        run: |
1927          cmake -B build \
1928            -DLLAMA_OPENSSL=OFF \
1929            -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
1930            -DGGML_OPENMP=OFF \
1931            -DLLAMA_BUILD_EXAMPLES=ON \
1932            -DLLAMA_BUILD_TOOLS=ON \
1933            -DLLAMA_BUILD_TESTS=OFF \
1934            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
1935            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
1936            -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
1937            -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
1938            -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14
1939
1940          cmake --build build --config ${{ matrix.build_type }} -j $(nproc)
1941
1942      - name: Test
1943        id: cmake_test
1944        run: |
1945          cd build
1946          ctest -L main --verbose --timeout 900
1947
1948
1949  ubuntu-llguidance-riscv64-native:
1950    runs-on: RISCV64
1951    steps:
1952      - name: Install dependencies
1953        run: |
1954          sudo apt-get update
1955
1956          # Install necessary packages
1957          sudo apt-get install -y libatomic1 libtsan2 gcc-14 g++-14 rustup cmake build-essential wget ccache git-lfs
1958
1959          # Set gcc-14 and g++-14 as the default compilers
1960          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100
1961          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100
1962          sudo ln -sf /usr/bin/gcc-14 /usr/bin/gcc
1963          sudo ln -sf /usr/bin/g++-14 /usr/bin/g++
1964
1965          # Install Rust stable version
1966          rustup install stable
1967          rustup default stable
1968
1969          git lfs install
1970
1971      - name: GCC version check
1972        run: |
1973          gcc --version
1974          g++ --version
1975
1976      - name: Clone
1977        id: checkout
1978        uses: actions/checkout@v6
1979
1980      - name: Setup ccache
1981        run: |
1982          export CCACHE_DIR="$HOME/.ccache/llguidance-riscv64"
1983          mkdir -p "$CCACHE_DIR"
1984
1985          ccache --set-config=max_size=5G
1986          ccache --set-config=compression=true
1987          ccache --set-config=compression_level=6
1988          ccache --set-config=cache_dir="$CCACHE_DIR"
1989          ccache --set-config=sloppiness=file_macro,time_macros,include_file_mtime,include_file_ctime
1990          ccache --set-config=hash_dir=false
1991
1992          echo "CCACHE_DIR=$CCACHE_DIR" >> $GITHUB_ENV
1993          echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
1994
1995      - name: Build
1996        id: cmake_build
1997        run: |
1998          cmake -B build \
1999            -DLLAMA_OPENSSL=OFF \
2000            -DCMAKE_BUILD_TYPE=Release \
2001            -DGGML_OPENMP=OFF \
2002            -DLLAMA_BUILD_EXAMPLES=ON \
2003            -DLLAMA_BUILD_TOOLS=ON \
2004            -DLLAMA_BUILD_TESTS=OFF \
2005            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
2006            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
2007            -DLLAMA_LLGUIDANCE=ON \
2008            -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
2009            -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14
2010
2011          cmake --build build --config Release -j $(nproc)
2012
2013      - name: Test
2014        id: cmake_test
2015        run: |
2016          cd build
2017          ctest -L main --verbose --timeout 900
2018
2019
2020  ubuntu-cmake-rpc-riscv64-native:
2021    runs-on: RISCV64
2022
2023    continue-on-error: true
2024
2025    steps:
2026      - name: Install dependencies
2027        run: |
2028          sudo apt-get update
2029
2030          # Install necessary packages
2031          sudo apt-get install -y libatomic1 libtsan2 gcc-14 g++-14 rustup cmake build-essential libssl-dev wget ccache git-lfs
2032
2033          # Set gcc-14 and g++-14 as the default compilers
2034          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100
2035          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100
2036          sudo ln -sf /usr/bin/gcc-14 /usr/bin/gcc
2037          sudo ln -sf /usr/bin/g++-14 /usr/bin/g++
2038
2039          # Install Rust stable version
2040          rustup install stable
2041          rustup default stable
2042
2043          git lfs install
2044
2045      - name: GCC version check
2046        run: |
2047          gcc --version
2048          g++ --version
2049
2050      - name: Clone
2051        id: checkout
2052        uses: actions/checkout@v6
2053
2054      - name: Setup ccache
2055        run: |
2056          export CCACHE_DIR="$HOME/.ccache/rpc-riscv64"
2057          mkdir -p "$CCACHE_DIR"
2058
2059          ccache --set-config=max_size=5G
2060          ccache --set-config=compression=true
2061          ccache --set-config=compression_level=6
2062          ccache --set-config=cache_dir="$CCACHE_DIR"
2063          ccache --set-config=sloppiness=file_macro,time_macros,include_file_mtime,include_file_ctime
2064          ccache --set-config=hash_dir=false
2065
2066          echo "CCACHE_DIR=$CCACHE_DIR" >> $GITHUB_ENV
2067          echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
2068
2069      - name: Build
2070        id: cmake_build
2071        run: |
2072          cmake -B build \
2073            -DCMAKE_BUILD_TYPE=Release \
2074            -DGGML_OPENMP=OFF \
2075            -DLLAMA_BUILD_EXAMPLES=ON \
2076            -DLLAMA_BUILD_TOOLS=ON \
2077            -DLLAMA_BUILD_TESTS=ON \
2078            -DCMAKE_C_COMPILER_LAUNCHER=ccache \
2079            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
2080            -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
2081            -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 \
2082            -DGGML_RPC=ON
2083
2084          cmake --build build --config Release -j $(nproc)
2085
2086      - name: Test
2087        id: cmake_test
2088        run: |
2089          cd build
2090          ctest -L main --verbose
2091
2092  ggml-ci-arm64-graviton4-kleidiai:
2093     runs-on: ah-ubuntu_22_04-c8g_8x
2094
2095     steps:
2096       - name: Clone
2097         id: checkout
2098         uses: actions/checkout@v6
2099
2100       - name: Dependencies
2101         id: depends
2102         run: |
2103           set -euxo pipefail
2104           sudo apt-get update
2105           sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \
2106           apt-get install -y \
2107            build-essential \
2108            python3-venv \
2109            gpg \
2110            wget \
2111            time \
2112            git-lfs
2113
2114           git lfs install
2115
2116           # install the latest cmake
2117           sudo install -d /usr/share/keyrings
2118           wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
2119            | gpg --dearmor \
2120            | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
2121           echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \
2122            | sudo tee /etc/apt/sources.list.d/kitware.list
2123           sudo apt-get update
2124           sudo apt-get install -y cmake
2125
2126       - name: ccache
2127         uses: ggml-org/ccache-action@v1.2.16
2128         with:
2129           key: ggml-ci-arm64-graviton4-kleidiai
2130           evict-old-files: 1d
2131           save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
2132
2133       - name: Test
2134         id: ggml-ci
2135         run: |
2136           GG_BUILD_KLEIDIAI=1 \
2137           GG_BUILD_EXTRA_TESTS_0=1 \
2138           bash ./ci/run.sh ./tmp/results ./tmp/mnt