Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 91 additions & 9 deletions ct/llama-cpp.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,35 @@ var_gpu="${var_gpu:-yes}"
var_unprivileged="${var_unprivileged:-1}"
var_testurl="${var_testurl:-https://github.com/community-scripts/DevScripts/issues/2124}"

# Asked here because the installer runs under lxc-attach, where it can never
# reach a terminal. Updates run inside the container and keep the recorded build.
if [[ -z "${var_backend:-}" ]] && command -v pveversion >/dev/null 2>&1 && [[ -t 0 ]]; then
LLAMA_BACKENDS=(
"auto" "Vulkan if a GPU is passed through, else CPU"
"cpu" "CPU only"
"vulkan" "Vulkan (Intel, AMD and NVIDIA GPUs)"
"cuda" "NVIDIA CUDA (12 or 13, from the host driver)"
)
if [[ "$(dpkg --print-architecture 2>/dev/null)" == "amd64" ]]; then
LLAMA_BACKENDS+=(
"rocm" "AMD ROCm"
"sycl-fp16" "Intel GPU via oneAPI SYCL (FP16)"
"sycl-fp32" "Intel GPU via oneAPI SYCL (FP32)"
"openvino" "Intel CPU/GPU via OpenVINO"
)
fi
var_backend=$(whiptail --backtitle "Proxmox VE Helper Scripts" --title "LLAMA.CPP BUILD" --default-item "auto" \
--menu "Choose the llama.cpp build:" $((${#LLAMA_BACKENDS[@]} / 2 + 8)) 72 $((${#LLAMA_BACKENDS[@]} / 2)) \
"${LLAMA_BACKENDS[@]}" 3>&1 1>&2 2>&3) || exit_script
fi

export var_backend="${var_backend:-}"
export var_model_repo="${var_model_repo:-}"
export var_ctx_size="${var_ctx_size:-}"
export var_offload_layers="${var_offload_layers:-}"
export var_api_key="${var_api_key:-}"
export var_port="${var_port:-}"

header_info "$APP"
variables
color
Expand All @@ -34,26 +63,79 @@ function update_script() {
exit
fi

if check_for_gh_release "llama-cpp" "ggml-org/llama.cpp"; then
# Follow the channel picked at install unless GH_INCLUDE_PRERELEASE is given
# for this run: the bNNNN prerelease builds, or the build a stable vX.Y.Z
# release names in its nightly-tag.txt (stable releases carry no binaries).
# Installs that predate the record all tracked the bNNNN builds, and reading
# them as stable would downgrade them.
GH_INCLUDE_PRERELEASE="${GH_INCLUDE_PRERELEASE:-$(cat /opt/llama-cpp_data/.prerelease 2>/dev/null || echo 1)}"
export GH_INCLUDE_PRERELEASE
LLAMA_RELEASE=""
if [[ "$GH_INCLUDE_PRERELEASE" == "1" ]]; then
if check_for_gh_release "llama-cpp" "ggml-org/llama.cpp" "" "" "b"; then
LLAMA_RELEASE="$CHECK_UPDATE_RELEASE"
fi
elif check_for_gh_release "llama-cpp-stable" "ggml-org/llama.cpp"; then
USE_ORIGINAL_FILENAME=true fetch_and_deploy_gh_release "llama-cpp-stable" "ggml-org/llama.cpp" "singlefile" "$CHECK_UPDATE_RELEASE" "/opt/llama-cpp_data" "nightly-tag.txt"
LLAMA_RELEASE="$(</opt/llama-cpp_data/nightly-tag.txt)"
fi

if [[ -n "$LLAMA_RELEASE" ]]; then
msg_info "Stopping Service"
systemctl stop llama-cpp
msg_ok "Stopped Service"

# Keep the build variant the install picked; the CPU tarball would otherwise
# overwrite a Vulkan install and quietly end GPU offload.
#
# rocm reads as vulkan: upstream publishes no Linux ROCm build, so anything
# recorded as rocm predates that being noticed and would otherwise fail
# every update on an asset that does not exist.
# overwrite a GPU build and quietly end offload.
LLAMA_BACKEND="$(cat /opt/llama-cpp_data/.backend 2>/dev/null || echo cpu)"
LLAMA_ARCH="$(arch_resolve x64 arm64)"
# Installs from before the ROCm build existed recorded rocm while actually
# running the Vulkan build.
[[ "$LLAMA_BACKEND" == "rocm" && ! -d /opt/llama-cpp_data/rocm ]] && LLAMA_BACKEND="vulkan"

LLAMA_CUDART_ASSET=""
case "$LLAMA_BACKEND" in
vulkan | rocm) LLAMA_ASSET="llama-*-bin-ubuntu-vulkan-$(arch_resolve x64 arm64).tar.gz" ;;
*) LLAMA_ASSET="llama-*-bin-ubuntu-$(arch_resolve x64 arm64).tar.gz" ;;
vulkan) LLAMA_ASSET="llama-*-bin-ubuntu-vulkan-${LLAMA_ARCH}.tar.gz" ;;
cuda-12 | cuda-13)
LLAMA_ASSET="llama-*-bin-ubuntu-cuda-${LLAMA_BACKEND#cuda-}.*-${LLAMA_ARCH}.tar.gz"
LLAMA_CUDART_ASSET="cudart-${LLAMA_ASSET}"
;;
rocm) LLAMA_ASSET="llama-*-bin-ubuntu-rocm-*-x64.tar.gz" ;;
sycl-fp16 | sycl-fp32) LLAMA_ASSET="llama-*-bin-ubuntu-${LLAMA_BACKEND}-x64.tar.gz" ;;
openvino) LLAMA_ASSET="llama-*-bin-ubuntu-openvino-*x64.tar.gz" ;;
*) LLAMA_ASSET="llama-*-bin-ubuntu-${LLAMA_ARCH}.tar.gz" ;;
esac

CLEAN_INSTALL=1 fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "latest" "/opt/llama-cpp" "$LLAMA_ASSET"
CLEAN_INSTALL=1 fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "$LLAMA_RELEASE" "/opt/llama-cpp" "$LLAMA_ASSET"
# The wipe above also removed the CUDA runtime, so it always goes back in.
if [[ -n "$LLAMA_CUDART_ASSET" ]]; then
rm -f ~/.llama-cpp-cudart
fetch_and_deploy_gh_release "llama-cpp-cudart" "ggml-org/llama.cpp" "prebuild" "$LLAMA_RELEASE" "/opt/llama-cpp" "$LLAMA_CUDART_ASSET"
fi
chmod +x /opt/llama-cpp/llama-*

case "$LLAMA_BACKEND" in
sycl-* | openvino)
fetch_and_deploy_gh_release "level-zero" "oneapi-src/level-zero" "binary" "latest" "" "libze1_*u24.04_amd64.deb" || true
if [[ -d /dev/dri ]]; then
fetch_and_deploy_gh_release "intel-level-zero-gpu" "intel/compute-runtime" "binary" "latest" "" "libze-intel-gpu1_*_amd64.deb" || true
fi
;;
rocm)
# The ROCm build links against the ROCm release it was built with, so
# follow it when upstream moves to a newer one.
ROCM_VERSION="$(jq -r 'if type == "array" then .[0] else . end | .assets[].name' "$TOOLS_GH_REL_JSON" 2>/dev/null | grep -oP 'bin-ubuntu-rocm-\K[0-9.]+(?=-x64)' | head -1)"
if [[ -n "$ROCM_VERSION" && "$ROCM_VERSION" != "$(cat /opt/llama-cpp_data/rocm/.version 2>/dev/null)" ]]; then
msg_info "Updating ROCm Runtime to ${ROCM_VERSION}"
$STD uv pip install --python /opt/llama-cpp_data/rocm --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries]==${ROCM_VERSION}.*"
echo "$ROCM_VERSION" >/opt/llama-cpp_data/rocm/.version
msg_ok "Updated ROCm Runtime to ${ROCM_VERSION}"
fi
;;
esac

echo "$GH_INCLUDE_PRERELEASE" >/opt/llama-cpp_data/.prerelease

msg_info "Starting Service"
systemctl start llama-cpp
msg_ok "Started Service"
Expand Down
170 changes: 142 additions & 28 deletions install/llama-cpp-install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,62 +13,147 @@ setting_up_container
network_check
update_os

var_backend="${var_backend:-auto}"
var_model_repo="${var_model_repo:-ggml-org/gemma-3-1b-it-GGUF}"
var_model_repo="${var_model_repo:-ggml-org/gemma-4-E2B-it-GGUF:Q4_0}"
var_port="${var_port:-8080}"
var_ctx_size="${var_ctx_size:-4096}"
var_offload_layers="${var_offload_layers:-}"
var_api_key="${var_api_key:-}"

# Which prebuilt upstream ships decides what the container can offload to, and
# for Linux that list is short: ubuntu-x64, ubuntu-arm64, ubuntu-vulkan-x64,
# ubuntu-vulkan-arm64, plus the Intel sycl and openvino builds. There is no
# Linux CUDA build and no Linux ROCm build either -- the only ROCm asset
# upstream publishes is win-rocm-*-x64.zip. Vulkan is therefore the GPU path for
# every vendor here, AMD included, and it reaches an AMD card through RADV.
#
# Asking for a ubuntu-rocm asset failed the install outright with "No asset
# matching 'llama--bin-ubuntu-rocm--x64.tar.gz' found" on any host where the
# engine had set up /dev/kfd.
# The Linux prebuilts upstream publishes (see .github/workflows/release.yml):
# cpu ubuntu-{x64,arm64}
# vulkan ubuntu-vulkan-{x64,arm64} any GPU vendor (AMD through RADV)
# cuda ubuntu-cuda-{12.x,13.x}-x64, ubuntu-cuda-13.x-arm64, plus a cudart tarball
# rocm ubuntu-rocm-<ver>-x64 needs the matching ROCm (TheRock) runtime
# sycl ubuntu-sycl-{fp16,fp32}-x64 Intel GPUs, needs oneAPI + Level Zero
# openvino ubuntu-openvino-<ver>-x64 Intel CPU/GPU, OpenVINO runtime is bundled
var_backend="${var_backend:-auto}"

if [[ "$var_backend" == "auto" ]]; then
if [[ -e /dev/kfd || -e /dev/nvidia0 || -d /dev/dri ]]; then
var_backend="vulkan"
else
var_backend="cpu"
fi
fi

# rocm was a documented value, so keep accepting it rather than failing anyone
# who pinned it -- but say where they are actually being sent.
if [[ "$var_backend" == "rocm" ]]; then
msg_warn "llama.cpp publishes no Linux ROCm build - using the Vulkan build, which drives an AMD card through RADV"
var_backend="vulkan"
fi
[[ "$var_backend" == "sycl" ]] && var_backend="sycl-fp16"

LLAMA_ARCH="$(arch_resolve x64 arm64)"

case "$var_backend" in
vulkan) LLAMA_ASSET="llama-*-bin-ubuntu-vulkan-${LLAMA_ARCH}.tar.gz" ;;
rocm | sycl-fp16 | sycl-fp32 | openvino)
if [[ "$LLAMA_ARCH" != "x64" ]]; then
msg_error "llama.cpp publishes the ${var_backend} build for x64 only"
exit 1
fi
;;
cpu | vulkan | cuda) ;;
*)
var_backend="cpu"
LLAMA_ASSET="llama-*-bin-ubuntu-${LLAMA_ARCH}.tar.gz"
msg_error "Unknown var_backend '${var_backend}' (expected auto, cpu, vulkan, cuda, rocm, sycl-fp16, sycl-fp32 or openvino)"
exit 1
;;
esac

msg_info "Installing Dependencies"
LLAMA_DEPS=(libgomp1)
[[ "$var_backend" == "vulkan" ]] && LLAMA_DEPS+=(libvulkan1 mesa-vulkan-drivers vulkan-tools)
case "$var_backend" in
vulkan) LLAMA_DEPS+=(libvulkan1 mesa-vulkan-drivers vulkan-tools) ;;
openvino) LLAMA_DEPS+=(libtbb12 ocl-icd-libopencl1) ;;
sycl-*) LLAMA_DEPS+=(ocl-icd-libopencl1) ;;
esac
$STD apt install -y "${LLAMA_DEPS[@]}"
msg_ok "Installed Dependencies"

setup_hwaccel

fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "latest" "/opt/llama-cpp" "$LLAMA_ASSET"
# SYCL and OpenVINO reach an Intel GPU through Level Zero. release.yml builds
# SYCL against the loader from oneapi-src/level-zero, so use the same package.
if [[ "$var_backend" == sycl-* || "$var_backend" == "openvino" ]]; then
fetch_and_deploy_gh_release "level-zero" "oneapi-src/level-zero" "binary" "latest" "" "libze1_*u24.04_amd64.deb"
if [[ -d /dev/dri ]]; then
fetch_and_deploy_gh_release "intel-level-zero-gpu" "intel/compute-runtime" "binary" "latest" "" "libze-intel-gpu1_*_amd64.deb" || true
fi
fi

# The SYCL tarball ships only llama.cpp itself; the DPC++/oneMKL/oneDNN/TBB
# libraries it links against come from oneAPI. release.yml builds with the full
# toolkit, but running needs only the runtime libraries.
if [[ "$var_backend" == sycl-* ]]; then
setup_deb822_repo \
"oneapi" \
"https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB" \
"https://apt.repos.intel.com/oneapi" \
"all" \
"main"
msg_info "Installing Intel oneAPI Runtime"
$STD apt install -y intel-oneapi-runtime-libs
# The runtime packages unpack under /opt/intel/oneapi without telling the
# dynamic linker, so register every directory that holds one of them.
find /opt/intel/oneapi -type f \( -name 'libsycl.so*' -o -name 'libur_loader.so*' -o -name 'libmkl_core.so*' \
-o -name 'libdnnl.so*' -o -name 'libtbb.so*' -o -name 'libsvml.so' -o -name 'libiomp5.so' -o -name 'libumf.so*' \) \
-printf '%h\n' 2>/dev/null | sort -u >/etc/ld.so.conf.d/oneapi-llama-cpp.conf
ldconfig
msg_ok "Installed Intel oneAPI Runtime"
fi

LLAMA_CUDART_ASSET=""
case "$var_backend" in
vulkan) LLAMA_ASSET="llama-*-bin-ubuntu-vulkan-${LLAMA_ARCH}.tar.gz" ;;
cuda)
# CUDA 13 needs a 580+ host driver; anything older gets the CUDA 12 build,
# which upstream publishes for x64 only.
NVIDIA_DRIVER_MAJOR="$(grep -oP '\d{3,}(?=\.\d+)' /proc/driver/nvidia/version 2>/dev/null | head -1)"
if [[ "$LLAMA_ARCH" == "x64" && -n "$NVIDIA_DRIVER_MAJOR" && "$NVIDIA_DRIVER_MAJOR" -lt 580 ]]; then
var_backend="cuda-12"
else
var_backend="cuda-13"
fi
[[ -z "$NVIDIA_DRIVER_MAJOR" ]] && msg_warn "No NVIDIA host driver visible - installing the CUDA 13 build, which needs driver 580 or newer"
LLAMA_ASSET="llama-*-bin-ubuntu-cuda-${var_backend#cuda-}.*-${LLAMA_ARCH}.tar.gz"
LLAMA_CUDART_ASSET="cudart-${LLAMA_ASSET}"
;;
rocm) LLAMA_ASSET="llama-*-bin-ubuntu-rocm-*-x64.tar.gz" ;;
sycl-fp16 | sycl-fp32) LLAMA_ASSET="llama-*-bin-ubuntu-${var_backend}-x64.tar.gz" ;;
openvino) LLAMA_ASSET="llama-*-bin-ubuntu-openvino-*x64.tar.gz" ;;
*) LLAMA_ASSET="llama-*-bin-ubuntu-${LLAMA_ARCH}.tar.gz" ;;
esac

# Only the bNNNN builds carry binaries, and upstream marks them prerelease. A
# stable vX.Y.Z release ships just nightly-tag.txt, naming the build it was cut
# from, so without GH_INCLUDE_PRERELEASE=1 that build is the one deployed.
mkdir -p /opt/llama-cpp_data/models
if [[ "${GH_INCLUDE_PRERELEASE:-0}" == "1" ]]; then
fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "latest" "/opt/llama-cpp" "$LLAMA_ASSET" "b"
LLAMA_RELEASE="$(cat ~/.llama-cpp)"
else
USE_ORIGINAL_FILENAME=true fetch_and_deploy_gh_release "llama-cpp-stable" "ggml-org/llama.cpp" "singlefile" "latest" "/opt/llama-cpp_data" "nightly-tag.txt"
LLAMA_RELEASE="$(</opt/llama-cpp_data/nightly-tag.txt)"
fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "$LLAMA_RELEASE" "/opt/llama-cpp" "$LLAMA_ASSET"
fi
if [[ -n "$LLAMA_CUDART_ASSET" ]]; then
fetch_and_deploy_gh_release "llama-cpp-cudart" "ggml-org/llama.cpp" "prebuild" "$LLAMA_RELEASE" "/opt/llama-cpp" "$LLAMA_CUDART_ASSET"
fi
LLAMA_LIBPATH="/opt/llama-cpp"

# The ROCm build carries no ROCm libraries. release.yml builds it against AMD's
# TheRock wheels, so install the same ROCm release of those as the runtime.
if [[ "$var_backend" == "rocm" ]]; then
ROCM_VERSION="$(jq -r --arg t "$LLAMA_RELEASE" 'if type == "array" then .[] else . end | select(.tag_name == $t) | .assets[].name' "$TOOLS_GH_REL_JSON" 2>/dev/null | grep -oP 'bin-ubuntu-rocm-\K[0-9.]+(?=-x64)' | head -1)"
if [[ -z "$ROCM_VERSION" ]]; then
msg_error "Could not read the ROCm release the llama.cpp ${LLAMA_RELEASE} build targets"
exit 1
fi
PYTHON_VERSION="3.12" setup_uv
msg_info "Installing ROCm ${ROCM_VERSION} Runtime"
$STD uv venv --python 3.12 /opt/llama-cpp_data/rocm
$STD uv pip install --python /opt/llama-cpp_data/rocm --index-url https://stable.repo.amd.com/rocm/whl-next/ "rocm[libraries]==${ROCM_VERSION}.*"
echo "$ROCM_VERSION" >/opt/llama-cpp_data/rocm/.version
LLAMA_LIBPATH="${LLAMA_LIBPATH}:$(/opt/llama-cpp_data/rocm/bin/rocm-sdk path --root)/lib"
msg_ok "Installed ROCm ${ROCM_VERSION} Runtime"
fi

msg_info "Configuring llama.cpp"
chmod +x /opt/llama-cpp/llama-*
mkdir -p /opt/llama-cpp_data/models
echo "$var_backend" >/opt/llama-cpp_data/.backend
echo "${GH_INCLUDE_PRERELEASE:-0}" >/opt/llama-cpp_data/.prerelease
if [[ -z "$var_offload_layers" ]]; then
[[ "$var_backend" == "cpu" ]] && var_offload_layers=0 || var_offload_layers=999
fi
Expand All @@ -80,11 +165,36 @@ LLAMA_ARG_PORT=${var_port}
#LLAMA_ARG_MODEL=/opt/llama-cpp_data/models/your-model.gguf
# ...or let llama-server pull one from Hugging Face on first start:
LLAMA_ARG_HF_REPO=${var_model_repo}
# Model switching: comment out LLAMA_ARG_HF_REPO (and LLAMA_ARG_MODEL) to serve
# every model in this folder and pick one per request from the web UI or API:
LLAMA_ARG_MODELS_DIR=/opt/llama-cpp_data/models
LLAMA_CACHE=/opt/llama-cpp_data/models
LLAMA_ARG_CTX_SIZE=${var_ctx_size}
LLAMA_ARG_N_GPU_LAYERS=${var_offload_layers}
EOF
case "$var_backend" in
sycl-*)
cat <<EOF >>/opt/llama-cpp.env
# SYCL: the persistent kernel cache speeds up restarts but can stop models
# from loading on some Intel GPUs, so it is off by default
#SYCL_CACHE_PERSISTENT=1
#SYCL_CACHE_DIR=/opt/llama-cpp_data/sycl-cache
#ONEAPI_DEVICE_SELECTOR=level_zero:0
EOF
;;
openvino)
OV_DEVICE="CPU"
compgen -G "/dev/dri/renderD*" >/dev/null && OV_DEVICE="GPU"
cat <<EOF >>/opt/llama-cpp.env
# OpenVINO device: CPU, GPU (GPU.0, GPU.1, ...) or NPU
GGML_OPENVINO_DEVICE=${OV_DEVICE}
GGML_OPENVINO_CACHE_DIR=/opt/llama-cpp_data/openvino-cache
EOF
;;
esac
# An empty LLAMA_ARG_API_KEY makes llama-server demand an empty bearer token.
if [[ -n "$var_api_key" ]]; then
echo "LLAMA_ARG_API_KEY=${var_api_key}" >>/opt/llama-cpp.env
Expand All @@ -95,10 +205,14 @@ chmod 600 /opt/llama-cpp.env
msg_ok "Configured llama.cpp (${var_backend} build)"

if [[ "$var_backend" != "cpu" ]]; then
if LD_LIBRARY_PATH=/opt/llama-cpp ldd /opt/llama-cpp/llama-server 2>/dev/null | grep -q "not found"; then
if LD_LIBRARY_PATH="$LLAMA_LIBPATH" ldd /opt/llama-cpp/llama-server /opt/llama-cpp/libggml-*.so 2>/dev/null | grep -q "not found"; then
msg_warn "The ${var_backend} build is missing shared libraries in this container - it will fall back to the CPU"
elif [[ "$var_backend" == "vulkan" ]] && ! vulkaninfo --summary >/dev/null 2>&1; then
msg_warn "Vulkan build installed but no Vulkan device is visible - check GPU passthrough and the driver libraries in the container"
elif [[ "$var_backend" == cuda-* ]] && ! ldconfig -p | grep -q 'libcuda[.]so'; then
msg_warn "CUDA build installed but the NVIDIA driver library (libcuda) is missing - check GPU passthrough"
elif [[ "$var_backend" == sycl-* || "$var_backend" == "rocm" ]] && [[ ! -d /dev/dri && ! -e /dev/kfd ]]; then
msg_warn "${var_backend} build installed but no GPU is passed through - it will fall back to the CPU"
fi
fi

Expand All @@ -114,7 +228,7 @@ Type=simple
User=root
WorkingDirectory=/opt/llama-cpp
EnvironmentFile=/opt/llama-cpp.env
Environment=LD_LIBRARY_PATH=/opt/llama-cpp
Environment=LD_LIBRARY_PATH=${LLAMA_LIBPATH}
ExecStart=/opt/llama-cpp/llama-server
Restart=on-failure
RestartSec=10
Expand Down
Loading
Loading