diff --git a/docker/Dockerfile.s390x b/docker/Dockerfile.s390x index c2adfafdee..b71e035e15 100644 --- a/docker/Dockerfile.s390x +++ b/docker/Dockerfile.s390x @@ -86,6 +86,29 @@ RUN --mount=type=cache,target=/root/.cache/uv \ mkdir -p /tmp/hf-xet/dist && \ cp dist/*.whl /tmp/hf-xet/dist/ +# Build LLVM 20 from source for llvmlite (system repos ship LLVM 21 which +# llvmlite v0.47 does not support; only SystemZ target is needed). +FROM base AS llvm20-build +ARG LLVM_VERSION=20.1.8 +WORKDIR /tmp +RUN microdnf install -y ninja-build gcc gcc-c++ python3 xz && \ + curl -LO https://github.com/llvm/llvm-project/releases/download/llvmorg-${LLVM_VERSION}/llvm-project-${LLVM_VERSION}.src.tar.xz && \ + tar -xf llvm-project-${LLVM_VERSION}.src.tar.xz && \ + cmake -G Ninja -S llvm-project-${LLVM_VERSION}.src/llvm -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/llvm20 \ + -DLLVM_TARGETS_TO_BUILD="SystemZ" \ + -DLLVM_ENABLE_RTTI=ON \ + -DLLVM_BUILD_TOOLS=OFF \ + -DLLVM_BUILD_UTILS=ON \ + -DLLVM_BUILD_EXAMPLES=OFF \ + -DLLVM_BUILD_TESTS=OFF \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_INCLUDE_EXAMPLES=OFF \ + -DLLVM_INCLUDE_BENCHMARKS=OFF && \ + ninja -C build install && \ + rm -rf build llvm-project-${LLVM_VERSION}.src* + # Build numba FROM python-install AS numba-builder @@ -96,11 +119,13 @@ WORKDIR /tmp # Clone all required dependencies RUN --mount=type=cache,target=/root/.cache/uv \ + --mount=type=bind,from=llvm20-build,source=/opt/llvm20,target=/opt/llvm20 \ microdnf install ninja-build gcc gcc-c++ -y && \ git clone --recursive https://github.com/numba/llvmlite.git -b v0.47.0 && \ git clone --recursive https://github.com/numba/numba.git -b ${NUMBA_VERSION} && \ cd llvmlite && \ uv pip install 'cmake<4' 'setuptools<70' numpy && \ + CMAKE_PREFIX_PATH=/opt/llvm20 LLVM_CONFIG=/opt/llvm20/bin/llvm-config \ python setup.py bdist_wheel && \ cd ../numba && \ if ! grep '#include "dynamic_annotations.h"' numba/_dispatcher.cpp; then \ @@ -158,7 +183,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \ NUMBA_WHL_FILE=$(ls /tmp/numba-wheels/*.whl) && \ OPENCV_WHL_FILE=$(ls /tmp/opencv-wheels/*.whl) && \ uv pip install -v \ - $ARROW_WHL_FILE \ $VISION_WHL_FILE \ $HF_XET_WHL_FILE \ $LLVM_WHL_FILE \ diff --git a/docs/getting_started/installation/cpu.md b/docs/getting_started/installation/cpu.md index 8b3605e855..1534d4d9e6 100644 --- a/docs/getting_started/installation/cpu.md +++ b/docs/getting_started/installation/cpu.md @@ -315,7 +315,7 @@ vLLM CPU supports data parallel (DP), tensor parallel (TP) and pipeline parallel - vLLM CPU supports quantizations: - AWQ (x86 only) - GPTQ (x86 only) - - compressed-tensor INT8 W8A8 (x86, s390x) + - compressed-tensor INT8 W8A8 (x86 only) ### Why do I see `get_mempolicy: Operation not permitted` when running in Docker? diff --git a/docs/getting_started/installation/cpu.s390x.inc.md b/docs/getting_started/installation/cpu.s390x.inc.md index 15baa487c2..407e13baf7 100644 --- a/docs/getting_started/installation/cpu.s390x.inc.md +++ b/docs/getting_started/installation/cpu.s390x.inc.md @@ -11,7 +11,7 @@ Currently, the CPU implementation for s390x architecture supports FP32, BF16 and - OS: `Linux` - SDK: `gcc/g++ >= 14.0.0` or later with Command Line Tools - Instruction Set Architecture (ISA): VXE support is required. Works with Z14 and above. -- Build install python packages: `torchvision`, `llvmlite`, `numba`, `pyarrow (for testing)`, `opencv-headless` +- Build from source python packages (no pre-built s390x wheels): `torchvision`, `llvmlite`, `numba`, `opencv-python-headless`, `hf-xet` --8<-- [end:requirements] --8<-- [start:set-up-using-python] @@ -28,13 +28,24 @@ Install the following packages from the package manager before building the vLLM ```bash dnf install -y \ - which procps findutils tar vim git gcc-toolset-14 gcc-toolset-14-binutils gcc-toolset-14-libatomic-devel zlib-devel \ + which procps findutils tar vim git patch xz ninja-build \ + gcc-toolset-14 gcc-toolset-14-binutils gcc-toolset-14-libatomic-devel zlib-devel \ libjpeg-turbo-devel libtiff-devel libpng-devel libwebp-devel freetype-devel harfbuzz-devel \ openssl-devel openblas openblas-devel autoconf automake libtool cmake numpy libsndfile \ clang llvm-devel llvm-static clang-devel ``` -Install rust>=1.80 which is needed for `outlines-core` and `uvloop` python packages installation. +Build and install `numactl` from source: + +```bash +curl -LO https://github.com/numactl/numactl/archive/refs/tags/v2.0.19.tar.gz +tar -xvzf v2.0.19.tar.gz +cd numactl-2.0.19 +./autogen.sh && ./configure && make && make install +cd .. +``` + +Install rust>=1.80 which is needed for `outlines-core`, `uvloop`, and `hf-xet` python packages installation. ```bash curl https://sh.rustup.rs -sSf | sh -s -- -y && \ @@ -44,26 +55,79 @@ curl https://sh.rustup.rs -sSf | sh -s -- -y && \ Execute the following commands to build and install vLLM from source. !!! tip - Please build the following dependencies, `torchvision`, `llvmlite`, `numba`, `llguidance`, `pyarrow`, `opencv-headless` from source before building vLLM. + Pre-built wheels are not available for s390x for the following packages. Build them from source before building vLLM: `torchvision`, `llvmlite`, `numba`, `opencv-python-headless`, `hf-xet`. + See `docker/Dockerfile.s390x` for exact versions and build commands used in each multi-stage build. + +!!! note "LLVM 20 required for llvmlite" + `llvmlite v0.47` requires LLVM 20, but UBI 9.6 repos ship LLVM 21 which is + not compatible. You must build LLVM 20 from source before building `llvmlite`: + + ```bash + curl -LO https://github.com/llvm/llvm-project/releases/download/llvmorg-20.1.8/llvm-project-20.1.8.src.tar.xz + tar -xf llvm-project-20.1.8.src.tar.xz + cmake -G Ninja -S llvm-project-20.1.8.src/llvm -B llvm-build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/opt/llvm20 \ + -DLLVM_TARGETS_TO_BUILD="SystemZ" \ + -DLLVM_ENABLE_RTTI=ON \ + -DLLVM_BUILD_TOOLS=OFF \ + -DLLVM_BUILD_UTILS=ON \ + -DLLVM_BUILD_EXAMPLES=OFF \ + -DLLVM_BUILD_TESTS=OFF \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_INCLUDE_EXAMPLES=OFF \ + -DLLVM_INCLUDE_BENCHMARKS=OFF + ninja -C llvm-build install + ``` + + Then build `llvmlite` pointing to LLVM 20: + + ```bash + CMAKE_PREFIX_PATH=/opt/llvm20 LLVM_CONFIG=/opt/llvm20/bin/llvm-config \ + python setup.py bdist_wheel + ``` ```bash - uv pip install -v \ - -r requirements/build/cpu.txt \ - -r requirements/cpu.txt \ - --torch-backend cpu \ - --index-strategy unsafe-best-match && \ - VLLM_TARGET_DEVICE=cpu python setup.py bdist_wheel && \ - uv pip install dist/*.whl +uv pip install -v \ + /path/to/torchvision.whl \ + /path/to/llvmlite.whl \ + /path/to/numba.whl \ + /path/to/opencv_python_headless.whl \ + /path/to/hf_xet.whl \ + -r requirements/build/cpu.txt \ + -r requirements/cpu.txt \ + --torch-backend cpu \ + --index-strategy unsafe-best-match && \ +VLLM_TARGET_DEVICE=cpu VLLM_CPU_MOE_PREPACK=0 python setup.py bdist_wheel && \ + uv pip install dist/*.whl ``` ??? console "pip" ```bash - pip install -v \ - --extra-index-url https://download.pytorch.org/whl/cpu \ - -r requirements/build/cpu.txt \ - -r requirements/cpu.txt \ - VLLM_TARGET_DEVICE=cpu python setup.py bdist_wheel && \ - pip install dist/*.whl + pip install -v \ + --extra-index-url https://download.pytorch.org/whl/cpu \ + /path/to/torchvision.whl \ + /path/to/llvmlite.whl \ + /path/to/numba.whl \ + /path/to/opencv_python_headless.whl \ + /path/to/hf_xet.whl \ + -r requirements/build/cpu.txt \ + -r requirements/cpu.txt && \ + VLLM_TARGET_DEVICE=cpu VLLM_CPU_MOE_PREPACK=0 python setup.py bdist_wheel && \ + pip install dist/*.whl + ``` + +!!! warning "Protobuf workaround for s390x" + The C++ protobuf extension crashes on s390x. After installation, set the + following environment variable and remove the C++ extensions: + + ```bash + export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python + + # Remove C++ protobuf extensions that crash on s390x + SITE_PKGS=$(python -c "import site; print(site.getsitepackages()[0])") + rm -rf "$SITE_PKGS/google/_upb/"*.so \ + "$SITE_PKGS/google/protobuf/pyext/"*.so 2>/dev/null || true ``` --8<-- [end:build-wheel-from-source] @@ -80,19 +144,20 @@ docker build -f docker/Dockerfile.s390x \ # Launch OpenAI server docker run --rm \ - --privileged true \ + --security-opt seccomp=unconfined \ + --cap-add SYS_NICE \ --shm-size 4g \ -p 8000:8000 \ -e VLLM_CPU_KVCACHE_SPACE= \ -e VLLM_CPU_OMP_THREADS_BIND= \ vllm-cpu-env \ --model meta-llama/Llama-3.2-1B-Instruct \ - --dtype float \ + --dtype bfloat16 \ other vLLM OpenAI server arguments ``` !!! tip - An alternative of `--privileged true` is `--cap-add SYS_NICE --security-opt seccomp=unconfined`. + Alternatively, `--privileged=true` also works but is broader and not generally recommended. --8<-- [end:build-image-from-source] --8<-- [start:extra-information]