From 1ba66d00f5095feac585f6385055c7308752d7be Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 23 Aug 2025 22:36:39 +0200 Subject: [PATCH] feat: bundle python inside backends (#6123) * feat(backends): bundle python Signed-off-by: Ettore Di Giacinto * test ci Signed-off-by: Ettore Di Giacinto * vllm on self-hosted Signed-off-by: Ettore Di Giacinto * Add clang Signed-off-by: Ettore Di Giacinto * Try to fix it for Mac * Relocate links only when is portable Signed-off-by: Ettore Di Giacinto * Make sure to call macosPortableEnv Signed-off-by: Ettore Di Giacinto * Use self-hosted for vllm Signed-off-by: Ettore Di Giacinto * Fixups Signed-off-by: Ettore Di Giacinto * CI Signed-off-by: Ettore Di Giacinto --------- Signed-off-by: Ettore Di Giacinto --- .github/workflows/backend.yml | 6 +- backend/Dockerfile.python | 4 +- backend/python/common/libbackend.sh | 369 +++++++++++++----- backend/python/diffusers/requirements-cpu.txt | 1 + scripts/build/python-darwin.sh | 1 + 5 files changed, 286 insertions(+), 95 deletions(-) diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 6eb957c4f..cae602f21 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -191,7 +191,7 @@ jobs: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-nvidia-cuda-12-vllm' - runs-on: 'ubuntu-latest' + runs-on: 'arc-runner-set' base-image: "ubuntu:22.04" skip-drivers: 'false' backend: "vllm" @@ -313,7 +313,7 @@ jobs: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-rocm-hipblas-vllm' - runs-on: 'ubuntu-latest' + runs-on: 'arc-runner-set' base-image: "rocm/dev-ubuntu-22.04:6.4.3" skip-drivers: 'false' backend: "vllm" @@ -435,7 +435,7 @@ jobs: platforms: 'linux/amd64' tag-latest: 'auto' tag-suffix: '-gpu-intel-vllm' - runs-on: 'ubuntu-latest' + runs-on: 'arc-runner-set' base-image: "quay.io/go-skynet/intel-oneapi-base:latest" skip-drivers: 'false' backend: "vllm" diff --git a/backend/Dockerfile.python b/backend/Dockerfile.python index 04e8e0201..9850e9808 100644 --- a/backend/Dockerfile.python +++ b/backend/Dockerfile.python @@ -23,7 +23,7 @@ RUN apt-get update && \ libssl-dev \ git \ git-lfs \ - unzip \ + unzip clang \ upx-ucl \ curl python3-pip \ python-is-python3 \ @@ -116,7 +116,7 @@ COPY python/${BACKEND} /${BACKEND} COPY backend.proto /${BACKEND}/backend.proto COPY python/common/ /${BACKEND}/common -RUN cd /${BACKEND} && make +RUN cd /${BACKEND} && PORTABLE_PYTHON=true make FROM scratch ARG BACKEND=rerankers diff --git a/backend/python/common/libbackend.sh b/backend/python/common/libbackend.sh index 79430ad2d..f7536b8df 100644 --- a/backend/python/common/libbackend.sh +++ b/backend/python/common/libbackend.sh @@ -1,6 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail - -# init handles the setup of the library # # use the library by adding the following line to a script: # source $(dirname $0)/../common/libbackend.sh @@ -20,34 +20,179 @@ # You can switch between uv (conda-like) and pip installation methods by setting USE_PIP: # USE_PIP=true source $(dirname $0)/../common/libbackend.sh # +# ===================== user-configurable defaults ===================== +PYTHON_VERSION="${PYTHON_VERSION:-3.10}" # e.g. 3.10 / 3.11 / 3.12 / 3.13 +PYTHON_PATCH="${PYTHON_PATCH:-18}" # e.g. 18 -> 3.10.18 ; 13 -> 3.11.13 +PY_STANDALONE_TAG="${PY_STANDALONE_TAG:-20250818}" # release tag date +# Enable/disable bundling of a portable Python build +PORTABLE_PYTHON="${PORTABLE_PYTHON:-false}" -PYTHON_VERSION="${PYTHON_VERSION:-3.10}" +# If you want to fully pin the filename (including tuned CPU targets), set: +# PORTABLE_PY_FILENAME="cpython-3.10.18+20250818-x86_64_v3-unknown-linux-gnu-install_only.tar.gz" +: "${PORTABLE_PY_FILENAME:=}" +: "${PORTABLE_PY_SHA256:=}" # optional; if set we verify the download +# ===================================================================== # Default to uv if USE_PIP is not set -if [ "x${USE_PIP}" == "x" ]; then +if [ "x${USE_PIP:-}" == "x" ]; then USE_PIP=false fi +# ----------------------- helpers ----------------------- +function _is_musl() { + # detect musl (Alpine, etc) + if command -v ldd >/dev/null 2>&1; then + ldd --version 2>&1 | grep -qi musl && return 0 + fi + # busybox-ish fallback + if command -v getconf >/dev/null 2>&1; then + getconf GNU_LIBC_VERSION >/dev/null 2>&1 || return 0 + fi + return 1 +} + +function _triple() { + local os="" arch="" libc="gnu" + case "$(uname -s)" in + Linux*) os="unknown-linux" ;; + Darwin*) os="apple-darwin" ;; + MINGW*|MSYS*|CYGWIN*) os="pc-windows-msvc" ;; # best-effort for Git Bash + *) echo "Unsupported OS $(uname -s)"; exit 1;; + esac + + case "$(uname -m)" in + x86_64) arch="x86_64" ;; + aarch64|arm64) arch="aarch64" ;; + armv7l) arch="armv7" ;; + i686|i386) arch="i686" ;; + ppc64le) arch="ppc64le" ;; + s390x) arch="s390x" ;; + riscv64) arch="riscv64" ;; + *) echo "Unsupported arch $(uname -m)"; exit 1;; + esac + + if [[ "$os" == "unknown-linux" ]]; then + if _is_musl; then + libc="musl" + else + libc="gnu" + fi + echo "${arch}-${os}-${libc}" + else + echo "${arch}-${os}" + fi +} + +function _portable_dir() { + echo "${EDIR}/python" +} + +function _portable_bin() { + # python-build-standalone puts python in ./bin + echo "$(_portable_dir)/bin" +} + +function _portable_python() { + if [ -x "$(_portable_bin)/python3" ]; then + echo "$(_portable_bin)/python3" + else + echo "$(_portable_bin)/python" + fi +} + + +# macOS loader env for the portable CPython +_macosPortableEnv() { + if [ "$(uname -s)" = "Darwin" ]; then + export DYLD_LIBRARY_PATH="$(_portable_dir)/lib${DYLD_LIBRARY_PATH:+:${DYLD_LIBRARY_PATH}}" + export DYLD_FALLBACK_LIBRARY_PATH="$(_portable_dir)/lib${DYLD_FALLBACK_LIBRARY_PATH:+:${DYLD_FALLBACK_LIBRARY_PATH}}" + fi +} + +# Good hygiene on macOS for downloaded/extracted trees +_unquarantinePortablePython() { + if [ "$(uname -s)" = "Darwin" ]; then + command -v xattr >/dev/null 2>&1 && xattr -dr com.apple.quarantine "$(_portable_dir)" || true + fi +} + +# ------------------ ### PORTABLE PYTHON ------------------ +function ensurePortablePython() { + local pdir="$(_portable_dir)" + local pbin="$(_portable_bin)" + local pyexe + + if [ -x "${pbin}/python3" ] || [ -x "${pbin}/python" ]; then + _macosPortableEnv + return 0 + fi + + mkdir -p "${pdir}" + local triple="$(_triple)" + + local full_ver="${PYTHON_VERSION}.${PYTHON_PATCH}" + local fn="" + if [ -n "${PORTABLE_PY_FILENAME}" ]; then + fn="${PORTABLE_PY_FILENAME}" + else + # generic asset name: cpython-+--install_only.tar.gz + fn="cpython-${full_ver}+${PY_STANDALONE_TAG}-${triple}-install_only.tar.gz" + fi + + local url="https://github.com/astral-sh/python-build-standalone/releases/download/${PY_STANDALONE_TAG}/${fn}" + local tmp="${pdir}/${fn}" + echo "Downloading portable Python: ${fn}" + # curl with retries; fall back to wget if needed + if command -v curl >/dev/null 2>&1; then + curl -L --fail --retry 3 --retry-delay 1 -o "${tmp}" "${url}" + else + wget -O "${tmp}" "${url}" + fi + + if [ -n "${PORTABLE_PY_SHA256}" ]; then + echo "${PORTABLE_PY_SHA256} ${tmp}" | sha256sum -c - + fi + + echo "Extracting ${fn} -> ${pdir}" + # always a .tar.gz (we purposely choose install_only) + tar -xzf "${tmp}" -C "${pdir}" + rm -f "${tmp}" + + # Some archives nest a directory; if so, flatten to ${pdir} + # Find the first dir with a 'bin/python*' + local inner + inner="$(find "${pdir}" -type f -path "*/bin/python*" -maxdepth 3 2>/dev/null | head -n1 || true)" + if [ -n "${inner}" ]; then + local inner_root + inner_root="$(dirname "$(dirname "${inner}")")" # .../bin -> root + if [ "${inner_root}" != "${pdir}" ]; then + # move contents up one level + shopt -s dotglob + mv "${inner_root}/"* "${pdir}/" + rm -rf "${inner_root}" + shopt -u dotglob + fi + fi + + _unquarantinePortablePython + _macosPortableEnv + # Make sure it's runnable + pyexe="$(_portable_python)" + "${pyexe}" -V +} + +# init handles the setup of the library function init() { - # Name of the backend (directory name) BACKEND_NAME=${PWD##*/} - - # Path where all backends files are - MY_DIR=$(realpath `dirname $0`) - - # Build type + MY_DIR=$(realpath "$(dirname "$0")") BUILD_PROFILE=$(getBuildProfile) - # Environment directory EDIR=${MY_DIR} - - # Allow to specify a custom env dir for shared environments - if [ "x${ENV_DIR}" != "x" ]; then + if [ "x${ENV_DIR:-}" != "x" ]; then EDIR=${ENV_DIR} fi - # If a backend has defined a list of valid build profiles... - if [ ! -z "${LIMIT_TARGETS}" ]; then + if [ ! -z "${LIMIT_TARGETS:-}" ]; then isValidTarget=$(checkTargets ${LIMIT_TARGETS}) if [ ${isValidTarget} != true ]; then echo "${BACKEND_NAME} can only be used on the following targets: ${LIMIT_TARGETS}" @@ -58,6 +203,7 @@ function init() { echo "Initializing libbackend for ${BACKEND_NAME}" } + # getBuildProfile will inspect the system to determine which build profile is appropriate: # returns one of the following: # - cublas11 @@ -65,83 +211,139 @@ function init() { # - hipblas # - intel function getBuildProfile() { - # First check if we are a cublas build, and if so report the correct build profile - if [ x"${BUILD_TYPE}" == "xcublas" ]; then - if [ ! -z ${CUDA_MAJOR_VERSION} ]; then - # If we have been given a CUDA version, we trust it + if [ x"${BUILD_TYPE:-}" == "xcublas" ]; then + if [ ! -z "${CUDA_MAJOR_VERSION:-}" ]; then echo ${BUILD_TYPE}${CUDA_MAJOR_VERSION} else - # We don't know what version of cuda we are, so we report ourselves as a generic cublas echo ${BUILD_TYPE} fi return 0 fi - # If /opt/intel exists, then we are doing an intel/ARC build if [ -d "/opt/intel" ]; then echo "intel" return 0 fi - # If for any other values of BUILD_TYPE, we don't need any special handling/discovery - if [ -n ${BUILD_TYPE} ]; then + if [ -n "${BUILD_TYPE:-}" ]; then echo ${BUILD_TYPE} return 0 fi - # If there is no BUILD_TYPE set at all, set a build-profile value of CPU, we aren't building for any GPU targets echo "cpu" } + +# Make the venv relocatable: +# - rewrite venv/bin/python{,3} to relative symlinks into $(_portable_dir) +# - normalize entrypoint shebangs to /usr/bin/env python3 +_makeVenvPortable() { + local venv_dir="${EDIR}/venv" + local vbin="${venv_dir}/bin" + + [ -d "${vbin}" ] || return 0 + + # 1) Replace python symlinks with relative ones to ../../python/bin/python3 + # (venv/bin -> venv -> EDIR -> python/bin) + local rel_py='../../python/bin/python3' + + for name in python3 python; do + if [ -e "${vbin}/${name}" ] || [ -L "${vbin}/${name}" ]; then + rm -f "${vbin}/${name}" + fi + done + ln -s "${rel_py}" "${vbin}/python3" + ln -s "python3" "${vbin}/python" + + # 2) Rewrite shebangs of entry points to use env, so the venv is relocatable + # Only touch text files that start with #! and reference the current venv. + local ve_abs="${vbin}/python" + local sed_i=(sed -i) + # macOS/BSD sed needs a backup suffix; GNU sed doesn't. Make it portable: + if sed --version >/dev/null 2>&1; then + sed_i=(sed -i) + else + sed_i=(sed -i '') + fi + + for f in "${vbin}"/*; do + [ -f "$f" ] || continue + # Fast path: check first two bytes (#!) + head -c2 "$f" 2>/dev/null | grep -q '^#!' || continue + # Only rewrite if the shebang mentions the (absolute) venv python + if head -n1 "$f" | grep -Fq "${ve_abs}"; then + "${sed_i[@]}" '1s|^#!.*$|#!/usr/bin/env python3|' "$f" + chmod +x "$f" 2>/dev/null || true + fi + done +} + + # ensureVenv makes sure that the venv for the backend both exists, and is activated. # # This function is idempotent, so you can call it as many times as you want and it will # always result in an activated virtual environment function ensureVenv() { - if [ ! -d "${EDIR}/venv" ]; then - if [ "x${USE_PIP}" == "xtrue" ]; then - echo "Using pip and Python virtual environments" + local interpreter="" - # Use Python virtual environment with pip - interpreter="python3" - # if there is no python , call python${PYTHON_VERSION} - - if command -v python${PYTHON_VERSION} &> /dev/null; then - interpreter="python${PYTHON_VERSION}" - fi - echo "Using interpreter: ${interpreter}" - ${interpreter} -m venv ${EDIR}/venv - source ${EDIR}/venv/bin/activate - ${interpreter} -m pip install --upgrade pip - echo "Python virtual environment created" + if [ "x${PORTABLE_PYTHON}" == "xtrue" ]; then + ensurePortablePython + interpreter="$(_portable_python)" + else + # Prefer system python${PYTHON_VERSION}, else python3, else fall back to bundled + if command -v python${PYTHON_VERSION} >/dev/null 2>&1; then + interpreter="python${PYTHON_VERSION}" + elif command -v python3 >/dev/null 2>&1; then + interpreter="python3" else - echo "Using uv package manager" - uv venv --python ${PYTHON_VERSION} ${EDIR}/venv - echo "uv virtual environment created" + echo "No suitable system Python found, bootstrapping portable build..." + ensurePortablePython + interpreter="$(_portable_python)" fi fi - # Source if we are not already in a Virtual env - if [ "x${VIRTUAL_ENV}" != "x${EDIR}/venv" ]; then - source ${EDIR}/venv/bin/activate - echo "Python virtual environment activated" + + if [ ! -d "${EDIR}/venv" ]; then + if [ "x${USE_PIP}" == "xtrue" ]; then + "${interpreter}" -m venv --copies "${EDIR}/venv" + source "${EDIR}/venv/bin/activate" + "${interpreter}" -m pip install --upgrade pip + else + if [ "x${PORTABLE_PYTHON}" == "xtrue" ]; then + uv venv --python "${interpreter}" "${EDIR}/venv" + else + uv venv --python "${PYTHON_VERSION}" "${EDIR}/venv" + fi + fi + if [ "x${PORTABLE_PYTHON}" == "xtrue" ]; then + _makeVenvPortable + fi fi - echo "activated virtual environment has been ensured" + # We call it here to make sure that when we source a venv we can still use python as expected + if [ -x "$(_portable_python)" ]; then + _macosPortableEnv + fi + + if [ "x${VIRTUAL_ENV:-}" != "x${EDIR}/venv" ]; then + source "${EDIR}/venv/bin/activate" + fi } + function runProtogen() { ensureVenv - if [ "x${USE_PIP}" == "xtrue" ]; then pip install grpcio-tools else uv pip install grpcio-tools fi - pushd ${EDIR} - python3 -m grpc_tools.protoc -I../../ -I./ --python_out=. --grpc_python_out=. backend.proto - popd + pushd "${EDIR}" >/dev/null + # use the venv python (ensures correct interpreter & sys.path) + python -m grpc_tools.protoc -I../../ -I./ --python_out=. --grpc_python_out=. backend.proto + popd >/dev/null } + # installRequirements looks for several requirements files and if they exist runs the install for them in order # # - requirements-install.txt @@ -165,38 +367,30 @@ function runProtogen() { # installRequirements function installRequirements() { ensureVenv - - # These are the requirements files we will attempt to install, in order declare -a requirementFiles=( "${EDIR}/requirements-install.txt" "${EDIR}/requirements.txt" - "${EDIR}/requirements-${BUILD_TYPE}.txt" + "${EDIR}/requirements-${BUILD_TYPE:-}.txt" ) - if [ "x${BUILD_TYPE}" != "x${BUILD_PROFILE}" ]; then + if [ "x${BUILD_TYPE:-}" != "x${BUILD_PROFILE}" ]; then requirementFiles+=("${EDIR}/requirements-${BUILD_PROFILE}.txt") fi - - # if BUILD_TYPE is empty, we are a CPU build, so we should try to install the CPU requirements - if [ "x${BUILD_TYPE}" == "x" ]; then + if [ "x${BUILD_TYPE:-}" == "x" ]; then requirementFiles+=("${EDIR}/requirements-cpu.txt") fi - requirementFiles+=("${EDIR}/requirements-after.txt") - - if [ "x${BUILD_TYPE}" != "x${BUILD_PROFILE}" ]; then + if [ "x${BUILD_TYPE:-}" != "x${BUILD_PROFILE}" ]; then requirementFiles+=("${EDIR}/requirements-${BUILD_PROFILE}-after.txt") fi for reqFile in ${requirementFiles[@]}; do - if [ -f ${reqFile} ]; then + if [ -f "${reqFile}" ]; then echo "starting requirements install for ${reqFile}" if [ "x${USE_PIP}" == "xtrue" ]; then - # Use pip for installation - pip install ${EXTRA_PIP_INSTALL_FLAGS} --requirement ${reqFile} + pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --requirement "${reqFile}" else - # Use uv for installation - uv pip install ${EXTRA_PIP_INSTALL_FLAGS} --requirement ${reqFile} + uv pip install ${EXTRA_PIP_INSTALL_FLAGS:-} --requirement "${reqFile}" fi echo "finished requirements install for ${reqFile}" fi @@ -220,18 +414,18 @@ function installRequirements() { # - ${BACKEND_NAME}.py function startBackend() { ensureVenv - - if [ ! -z ${BACKEND_FILE} ]; then - exec ${EDIR}/venv/bin/python ${BACKEND_FILE} $@ + if [ ! -z "${BACKEND_FILE:-}" ]; then + exec "${EDIR}/venv/bin/python" "${BACKEND_FILE}" "$@" elif [ -e "${MY_DIR}/server.py" ]; then - exec ${EDIR}/venv/bin/python ${MY_DIR}/server.py $@ + exec "${EDIR}/venv/bin/python" "${MY_DIR}/server.py" "$@" elif [ -e "${MY_DIR}/backend.py" ]; then - exec ${EDIR}/venv/bin/python ${MY_DIR}/backend.py $@ + exec "${EDIR}/venv/bin/python" "${MY_DIR}/backend.py" "$@" elif [ -e "${MY_DIR}/${BACKEND_NAME}.py" ]; then - exec ${EDIR}/venv/bin/python ${MY_DIR}/${BACKEND_NAME}.py $@ + exec "${EDIR}/venv/bin/python" "${MY_DIR}/${BACKEND_NAME}.py" "$@" fi } + # runUnittests discovers and runs python unittests # # You can specify a specific test file to use by setting TEST_FILE before calling runUnittests. @@ -244,41 +438,36 @@ function startBackend() { # be default a file named test.py in the backends directory will be used function runUnittests() { ensureVenv - - if [ ! -z ${TEST_FILE} ]; then - testDir=$(dirname `realpath ${TEST_FILE}`) - testFile=$(basename ${TEST_FILE}) - pushd ${testDir} - python -m unittest ${testFile} - popd + if [ ! -z "${TEST_FILE:-}" ]; then + testDir=$(dirname "$(realpath "${TEST_FILE}")") + testFile=$(basename "${TEST_FILE}") + pushd "${testDir}" >/dev/null + python -m unittest "${testFile}" + popd >/dev/null elif [ -f "${MY_DIR}/test.py" ]; then - pushd ${MY_DIR} + pushd "${MY_DIR}" >/dev/null python -m unittest test.py - popd + popd >/dev/null else echo "no tests defined for ${BACKEND_NAME}" fi } + ################################################################################## # Below here are helper functions not intended to be used outside of the library # ################################################################################## # checkTargets determines if the current BUILD_TYPE or BUILD_PROFILE is in a list of valid targets function checkTargets() { - # Collect all provided targets into a variable and... targets=$@ - # ...convert it into an array declare -a targets=($targets) - for target in ${targets[@]}; do - if [ "x${BUILD_TYPE}" == "x${target}" ]; then - echo true - return 0 + if [ "x${BUILD_TYPE:-}" == "x${target}" ]; then + echo true; return 0 fi if [ "x${BUILD_PROFILE}" == "x${target}" ]; then - echo true - return 0 + echo true; return 0 fi done echo false diff --git a/backend/python/diffusers/requirements-cpu.txt b/backend/python/diffusers/requirements-cpu.txt index 3a579ee41..659d2b8e0 100644 --- a/backend/python/diffusers/requirements-cpu.txt +++ b/backend/python/diffusers/requirements-cpu.txt @@ -1,3 +1,4 @@ +--extra-index-url https://download.pytorch.org/whl/cpu git+https://github.com/huggingface/diffusers opencv-python transformers diff --git a/scripts/build/python-darwin.sh b/scripts/build/python-darwin.sh index 6166a2630..4b0a373e0 100644 --- a/scripts/build/python-darwin.sh +++ b/scripts/build/python-darwin.sh @@ -2,6 +2,7 @@ set -ex +export PORTABLE_PYTHON=true IMAGE_NAME="${IMAGE_NAME:-localai/llama-cpp-darwin}" mkdir -p backend-images make -C backend/python/${BACKEND}