File size: 2,723 Bytes
46def5b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | #!/usr/bin/env bash
# ----------------------------------------------------------------------------
# openPangu-2.0-Flash support + speed optimizations for llama.cpp v0.7.0
#
# Downloads the pinned upstream llama.cpp commit, applies the patch and
# prints build/run instructions.
#
# Usage: ./patch.sh
# ----------------------------------------------------------------------------
set -euo pipefail
LLAMA_COMMIT="620d2225cc436ab0b001e3200e590e3b286ef9ab" # llama.cpp release v0.7.0
SRC_DIR="llama.cpp-${LLAMA_COMMIT}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PATCH_FILE="${SCRIPT_DIR}/llama.cpp-openpangu-v0.7.0.diff"
if [ ! -f "${PATCH_FILE}" ]; then
echo "error: ${PATCH_FILE} not found (keep it next to this script)" >&2
exit 1
fi
cd "${SCRIPT_DIR}"
if [ ! -d "${SRC_DIR}" ]; then
echo "==> downloading llama.cpp @ ${LLAMA_COMMIT} (v0.7.0)"
if command -v curl >/dev/null 2>&1; then
curl -L --fail -o llama.tar.gz \
"https://codeload.github.com/ggml-org/llama.cpp/tar.gz/${LLAMA_COMMIT}"
elif command -v wget >/dev/null 2>&1; then
wget -O llama.tar.gz \
"https://codeload.github.com/ggml-org/llama.cpp/tar.gz/${LLAMA_COMMIT}"
else
echo "error: need curl or wget" >&2
exit 1
fi
echo "==> extracting"
tar xzf llama.tar.gz
rm -f llama.tar.gz
fi
cd "${SRC_DIR}"
echo "==> applying $(basename "${PATCH_FILE}")"
if command -v git >/dev/null 2>&1 && git apply --check "${PATCH_FILE}" 2>/dev/null; then
git apply "${PATCH_FILE}"
elif command -v patch >/dev/null 2>&1; then
patch -p1 --forward < "${PATCH_FILE}"
else
echo "error: need git or patch to apply the diff" >&2
exit 1
fi
echo
echo "==> done. build with:"
echo
echo " cd ${SRC_DIR}"
echo " cmake -B build -S . -DCMAKE_BUILD_TYPE=Release"
echo " cmake --build build -j --target llama-cli llama-server"
echo
echo " # CPU only (no GPU):"
echo " cmake -B build -S . -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=ON"
echo " # CUDA:"
echo " cmake -B build -S . -DCMAKE_BUILD_TYPE=Release -DGGML_CUDA=ON"
echo
echo "==> run (get the GGUF models at https://huggingface.co/mrexodia/openPangu-2.0-Flash-GGUF):"
echo
echo " ./build/bin/llama-cli -m openPangu-2.0-Flash-base-Q3_K_M.gguf -c 65536 --jinja -t \$(nproc)"
echo " ./build/bin/llama-server -m openPangu-2.0-Flash-base-Q3_K_M.gguf -c 65536 --jinja --host 0.0.0.0 --port 8080"
echo
echo " # optional multi-token-prediction self-speculative decoding (needs the split MTP head):"
echo " ./build/bin/llama-cli -m openPangu-2.0-Flash-base-Q3_K_M.gguf \\"
echo " --spec-draft-model openPangu-2.0-Flash-mtp-Q8_0.gguf --spec-type draft-mtp -c 65536 --jinja"
|