diff --git a/docker/Dockerfile.s390x b/docker/Dockerfile.s390x index 6d1c0c3452..c2adfafdee 100644 --- a/docker/Dockerfile.s390x +++ b/docker/Dockerfile.s390x @@ -37,37 +37,6 @@ RUN microdnf install -y \ python${PYTHON_VERSION}-devel python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-wheel && \ python${PYTHON_VERSION} -m venv $VIRTUAL_ENV && pip install --no-cache -U pip wheel uv && microdnf clean all -FROM python-install AS pyarrow - -# Build Apache Arrow -WORKDIR /tmp -RUN --mount=type=cache,target=/root/.cache/uv \ - git clone https://github.com/apache/arrow.git -b maint-19.0.1 && \ - cd arrow/cpp && \ - mkdir release && cd release && \ - cmake -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/usr/local \ - -DARROW_PYTHON=ON \ - -DARROW_PARQUET=ON \ - -DARROW_ORC=ON \ - -DARROW_FILESYSTEM=ON \ - -DARROW_WITH_LZ4=ON \ - -DARROW_WITH_ZSTD=ON \ - -DARROW_WITH_SNAPPY=ON \ - -DARROW_JSON=ON \ - -DARROW_CSV=ON \ - -DARROW_DATASET=ON \ - -DPROTOBUF_PROTOC_EXECUTABLE=/usr/bin/protoc \ - -DARROW_DEPENDENCY_SOURCE=BUNDLED \ - .. && \ - make -j$(nproc) && \ - make install && \ - cd ../../python && \ - export PYARROW_PARALLEL=4 && \ - export ARROW_BUILD_TYPE=release && \ - uv pip install -r requirements-build.txt && \ - python setup.py build_ext --build-type=$ARROW_BUILD_TYPE --bundle-arrow-cpp bdist_wheel - FROM python-install AS rust ENV CARGO_HOME=/root/.cargo ENV RUSTUP_HOME=/root/.rustup @@ -121,60 +90,18 @@ RUN --mount=type=cache,target=/root/.cache/uv \ FROM python-install AS numba-builder ARG MAX_JOBS -ARG NUMBA_VERSION=0.61.2 +ARG NUMBA_VERSION=0.65.0 WORKDIR /tmp # Clone all required dependencies RUN --mount=type=cache,target=/root/.cache/uv \ microdnf install ninja-build gcc gcc-c++ -y && \ - git clone --recursive https://github.com/llvm/llvm-project.git -b llvmorg-15.0.7 && \ - git clone --recursive https://github.com/numba/llvmlite.git -b v0.44.0 && \ + git clone --recursive https://github.com/numba/llvmlite.git -b v0.47.0 && \ git clone --recursive https://github.com/numba/numba.git -b ${NUMBA_VERSION} && \ - cd llvm-project && mkdir build && cd build && \ + cd llvmlite && \ uv pip install 'cmake<4' 'setuptools<70' numpy && \ - export PREFIX=/usr/local && CMAKE_ARGS="${CMAKE_ARGS} -DLLVM_ENABLE_PROJECTS=lld;libunwind;compiler-rt" \ - CFLAGS="$(echo $CFLAGS | sed 's/-fno-plt //g')" \ - CXXFLAGS="$(echo $CXXFLAGS | sed 's/-fno-plt //g')" \ - CMAKE_ARGS="${CMAKE_ARGS} -DFFI_INCLUDE_DIR=$PREFIX/include" \ - CMAKE_ARGS="${CMAKE_ARGS} -DFFI_LIBRARY_DIR=$PREFIX/lib" \ - cmake -DCMAKE_INSTALL_PREFIX="${PREFIX}" \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_LIBRARY_PATH="${PREFIX}" \ - -DLLVM_ENABLE_LIBEDIT=OFF \ - -DLLVM_ENABLE_LIBXML2=OFF \ - -DLLVM_ENABLE_RTTI=ON \ - -DLLVM_ENABLE_TERMINFO=OFF \ - -DLLVM_INCLUDE_BENCHMARKS=OFF \ - -DLLVM_INCLUDE_DOCS=OFF \ - -DLLVM_INCLUDE_EXAMPLES=OFF \ - -DLLVM_INCLUDE_GO_TESTS=OFF \ - -DLLVM_INCLUDE_TESTS=OFF \ - -DLLVM_INCLUDE_UTILS=ON \ - -DLLVM_INSTALL_UTILS=ON \ - -DLLVM_UTILS_INSTALL_DIR=libexec/llvm \ - -DLLVM_BUILD_LLVM_DYLIB=OFF \ - -DLLVM_LINK_LLVM_DYLIB=OFF \ - -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly \ - -DLLVM_ENABLE_FFI=ON \ - -DLLVM_ENABLE_Z3_SOLVER=OFF \ - -DLLVM_OPTIMIZED_TABLEGEN=ON \ - -DCMAKE_POLICY_DEFAULT_CMP0111=NEW \ - -DCOMPILER_RT_BUILD_BUILTINS=ON \ - -DCOMPILER_RT_BUILTINS_HIDE_SYMBOLS=OFF \ - -DCOMPILER_RT_BUILD_LIBFUZZER=OFF \ - -DCOMPILER_RT_BUILD_CRT=OFF \ - -DCOMPILER_RT_BUILD_MEMPROF=OFF \ - -DCOMPILER_RT_BUILD_PROFILE=OFF \ - -DCOMPILER_RT_BUILD_SANITIZERS=OFF \ - -DCOMPILER_RT_BUILD_XRAY=OFF \ - -DCOMPILER_RT_BUILD_GWP_ASAN=OFF \ - -DCOMPILER_RT_BUILD_ORC=OFF \ - -DCOMPILER_RT_INCLUDE_TESTS=OFF \ - ${CMAKE_ARGS} -GNinja ../llvm \ - && ninja install . && \ - # build llvmlite - cd ../../llvmlite && python setup.py bdist_wheel && \ + python setup.py bdist_wheel && \ cd ../numba && \ if ! grep '#include "dynamic_annotations.h"' numba/_dispatcher.cpp; then \ sed -i '/#include "internal\/pycore_atomic.h"/i\#include "dynamic_annotations.h"' numba/_dispatcher.cpp; \ @@ -192,21 +119,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \ cd opencv-python && \ python -m build --wheel --installer=uv --outdir /tmp/opencv-python/dist -## Todo(r3hankhan123): Remove guidance-builder stage once vLLM upgrades to new version of llguidance that fixes s390x issues. See https://github.com/guidance-ai/llguidance/issues/330 -FROM python-install AS guidance-builder -WORKDIR /tmp -ENV CARGO_HOME=/root/.cargo -ENV RUSTUP_HOME=/root/.rustup -ENV PATH="$CARGO_HOME/bin:$RUSTUP_HOME/bin:$PATH" -RUN --mount=type=cache,target=/root/.cache/uv \ - --mount=type=bind,from=rust,source=/root/.cargo,target=/root/.cargo,rw \ - --mount=type=bind,from=rust,source=/root/.rustup,target=/root/.rustup,rw \ - git clone https://github.com/guidance-ai/llguidance.git && \ - cd llguidance && \ - git checkout s390x-fix-v2 && \ - uv pip install maturin && \ - python -m maturin build --release --out dist --compatibility linux - # # Final build stage FROM python-install AS vllm-cpu ARG PYTHON_VERSION @@ -235,20 +147,16 @@ RUN --mount=type=bind,from=numa-build,src=/tmp/numactl-2.0.19,target=/numactl \ RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=bind,from=rust,source=/root/.cargo,target=/root/.cargo,rw \ --mount=type=bind,from=rust,source=/root/.rustup,target=/root/.rustup,rw \ - --mount=type=bind,from=pyarrow,source=/tmp/arrow/python/dist,target=/tmp/arrow-wheels \ --mount=type=bind,from=torch-vision,source=/tmp/vision/dist,target=/tmp/vision-wheels/ \ --mount=type=bind,from=hf-xet-builder,source=/tmp/hf-xet/dist,target=/tmp/hf-xet-wheels/ \ --mount=type=bind,from=numba-builder,source=/tmp/llvmlite/dist,target=/tmp/llvmlite-wheels/ \ --mount=type=bind,from=numba-builder,source=/tmp/numba/dist,target=/tmp/numba-wheels/ \ --mount=type=bind,from=opencv-builder,source=/tmp/opencv-python/dist,target=/tmp/opencv-wheels/ \ - --mount=type=bind,from=guidance-builder,source=/tmp/llguidance/dist,target=/tmp/guidance-wheels/ \ - ARROW_WHL_FILE=$(ls /tmp/arrow-wheels/*.whl) && \ VISION_WHL_FILE=$(ls /tmp/vision-wheels/*.whl) && \ HF_XET_WHL_FILE=$(ls /tmp/hf-xet-wheels/*.whl) && \ LLVM_WHL_FILE=$(ls /tmp/llvmlite-wheels/*.whl) && \ NUMBA_WHL_FILE=$(ls /tmp/numba-wheels/*.whl) && \ OPENCV_WHL_FILE=$(ls /tmp/opencv-wheels/*.whl) && \ - GUIDANCE_WHL_FILE=$(ls /tmp/guidance-wheels/*.whl) && \ uv pip install -v \ $ARROW_WHL_FILE \ $VISION_WHL_FILE \ @@ -256,7 +164,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \ $LLVM_WHL_FILE \ $NUMBA_WHL_FILE \ $OPENCV_WHL_FILE \ - $GUIDANCE_WHL_FILE \ --torch-backend cpu \ --index-strategy unsafe-best-match \ -r requirements/build/cpu.txt \ diff --git a/requirements/common.txt b/requirements/common.txt index 1652480c22..30332ed448 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -21,7 +21,7 @@ pillow # Required for image processing prometheus-fastapi-instrumentator >= 8.0.0 # v8 unblocks starlette >= 1.0 tiktoken >= 0.6.0 # Required for DBRX tokenizer lm-format-enforcer == 0.11.3 -llguidance >= 1.7.0, < 1.8.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64" or platform_machine == "ppc64le" +llguidance >= 1.7.0, < 1.8.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "s390x" outlines_core == 0.2.14 # required for outlines backend disk cache diskcache == 5.6.3 diff --git a/vllm/distributed/device_communicators/cpu_communicator.py b/vllm/distributed/device_communicators/cpu_communicator.py index b8d9d6c53d..9ec4b72f80 100644 --- a/vllm/distributed/device_communicators/cpu_communicator.py +++ b/vllm/distributed/device_communicators/cpu_communicator.py @@ -33,6 +33,7 @@ class CpuCommunicator(DeviceCommunicatorBase): current_platform.get_cpu_architecture() == CpuArchEnum.X86 or current_platform.get_cpu_architecture() == CpuArchEnum.ARM or current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC + or current_platform.get_cpu_architecture() == CpuArchEnum.S390X ) and hasattr(torch.ops._C, "init_shm_manager") and (unique_name.startswith("tp") or unique_name.startswith("pp")) diff --git a/vllm/platforms/cpu.py b/vllm/platforms/cpu.py index ed7cec519a..0e805794f5 100644 --- a/vllm/platforms/cpu.py +++ b/vllm/platforms/cpu.py @@ -247,7 +247,12 @@ class CpuPlatform(Platform): if ( platform.system() == "Linux" and cpu_architecture - in (CpuArchEnum.ARM, CpuArchEnum.POWERPC, CpuArchEnum.X86) + in ( + CpuArchEnum.ARM, + CpuArchEnum.POWERPC, + CpuArchEnum.X86, + CpuArchEnum.S390X, + ) and not ( "libomp" in ld_preload_str or "libgomp" in ld_preload_str diff --git a/vllm/utils/cpu_resource_utils.py b/vllm/utils/cpu_resource_utils.py index 5543f4b6b0..53e7461065 100644 --- a/vllm/utils/cpu_resource_utils.py +++ b/vllm/utils/cpu_resource_utils.py @@ -3,6 +3,7 @@ import json import os +import platform import subprocess import sys from dataclasses import dataclass @@ -216,9 +217,14 @@ def _get_cpu_list() -> list[LogicalCPUInfo]: # For MacOS, no user-level CPU affinity and SMT, return all CPUs return _synthesize_cpu_list() - lscpu_output = subprocess.check_output( - "lscpu --json --extended=CPU,CORE,NODE --online", shell=True, text=True - ) + if platform.machine() == "s390x": + lscpu_output = subprocess.check_output( + "lscpu -J -e=CPU,CORE,NODE,SOCKET,BOOK", shell=True, text=True + ) + else: + lscpu_output = subprocess.check_output( + "lscpu --json --extended=CPU,CORE,NODE --online", shell=True, text=True + ) # For platforms without NUMA, map bare `-` node to 0 so non-NUMA # systems keep the existing behavior from #39781. @@ -233,11 +239,50 @@ def _get_cpu_list() -> list[LogicalCPUInfo]: lscpu_output, ) + if platform.machine() == "s390x": + # On s390x, use the best topology level as the abstract CPU group + # key via numa_node. The hierarchy is book > socket > core; prefer + # book, fall back to socket if only one book exists. + lscpu_output = re.sub(r'"book":\s*-\s*(,|\n|\})', r'"book": 0\1', lscpu_output) + lscpu_output = re.sub( + r'"socket":\s*-\s*(,|\n|\})', r'"socket": 0\1', lscpu_output + ) + + raw_cpus = json.loads(lscpu_output)["cpus"] + book_values = set() + socket_values = set() + for entry in raw_cpus: + book_values.add(LogicalCPUInfo._int(str(entry.get("book", "-1")))) + socket_values.add(LogicalCPUInfo._int(str(entry.get("socket", "-1")))) + + # Pick the best partitioning level: book first, then socket + if len(book_values - {-1}) > 1: + group_key = "book" + elif len(socket_values - {-1}) > 1: + group_key = "socket" + else: + group_key = None + + if group_key is not None: + logical_book_socket_list: list[LogicalCPUInfo] = [] + for entry in raw_cpus: + cpu_id = LogicalCPUInfo._int(str(entry.get("cpu", "-1"))) + core = LogicalCPUInfo._int(str(entry.get("core", "-1"))) + group = LogicalCPUInfo._int(str(entry.get(group_key, "-1"))) + if -1 not in (cpu_id, core, group): + logical_book_socket_list.append( + LogicalCPUInfo(id=cpu_id, physical_core=core, numa_node=group) + ) + + if not logical_book_socket_list: + return _synthesize_cpu_list() + return logical_book_socket_list + logical_cpu_list: list[LogicalCPUInfo] = json.loads( lscpu_output, object_hook=LogicalCPUInfo.json_decoder )["cpus"] - # Filter CPUs with invalid attributes + # Filter CPUs with invalid attributes (only require id, core, node) logical_cpu_list = [ x for x in logical_cpu_list if -1 not in (x.id, x.physical_core, x.numa_node) ] diff --git a/vllm/utils/ompmultiprocessing.py b/vllm/utils/ompmultiprocessing.py index 711f527355..0a704e6d26 100644 --- a/vllm/utils/ompmultiprocessing.py +++ b/vllm/utils/ompmultiprocessing.py @@ -204,6 +204,14 @@ class OMPProcessManager: logical_cpu_list = cr_utils.get_allowed_cpu_list() local_world_size = self.local_world_size + + # On s390x, numa_node is remapped to book IDs (the best CPU + # partitioning level). Derive topology domains from the CPU list + # since they no longer correspond to physical memory nodes. + cpu_arch = current_platform.get_cpu_architecture() + if cpu_arch == CpuArchEnum.S390X: + allowed_numa_nodes = sorted(set(cpu.numa_node for cpu in logical_cpu_list)) + assert ( len(allowed_numa_nodes) >= local_world_size or self.simulate_multi_node ), ( diff --git a/vllm/v1/worker/cpu_worker.py b/vllm/v1/worker/cpu_worker.py index 2433bc8a10..0a52ceaa3a 100644 --- a/vllm/v1/worker/cpu_worker.py +++ b/vllm/v1/worker/cpu_worker.py @@ -55,9 +55,22 @@ class CPUWorker(Worker): allowed_memory_nodes, ) - torch.ops._C.init_cpu_memory_env([cpu_core.numa_node]) + # On s390x, numa_node may be a synthetic book ID that doesn't + # correspond to a real memory node. Fall back to first visible node. + if cpu_core.numa_node in allowed_memory_nodes: + memory_node = cpu_core.numa_node + else: + logger.warning( + "CPU group key %s is not a valid memory node. " + "Falling back to memory node %s.", + cpu_core.numa_node, + allowed_memory_nodes[0], + ) + memory_node = allowed_memory_nodes[0] - memory_status = get_memory_node_info(cpu_core.numa_node) + torch.ops._C.init_cpu_memory_env([memory_node]) + + memory_status = get_memory_node_info(memory_node) memory_fraction = vllm_config.cache_config.gpu_memory_utilization self.requested_cpu_memory = math.ceil( memory_status.total_memory * memory_fraction