[CPU] Create Proper Numa topology for s390x (#40714)

Signed-off-by: Rehan Khan <Rehan.Khan7@ibm.com>
Co-authored-by: Li, Jiang <jiang1.li@intel.com>
This commit is contained in:
Rehan Khan
2026-07-13 12:58:43 +08:00
committed by GitHub
co-authored by Li, Jiang <jiang1.li@intel.com>
parent 8c5dafcd09
commit 9e57de7197
7 changed files with 84 additions and 105 deletions
+4 -97
View File
@@ -37,37 +37,6 @@ RUN microdnf install -y \
python${PYTHON_VERSION}-devel python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-wheel && \
python${PYTHON_VERSION} -m venv $VIRTUAL_ENV && pip install --no-cache -U pip wheel uv && microdnf clean all
FROM python-install AS pyarrow
# Build Apache Arrow
WORKDIR /tmp
RUN --mount=type=cache,target=/root/.cache/uv \
git clone https://github.com/apache/arrow.git -b maint-19.0.1 && \
cd arrow/cpp && \
mkdir release && cd release && \
cmake -DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=/usr/local \
-DARROW_PYTHON=ON \
-DARROW_PARQUET=ON \
-DARROW_ORC=ON \
-DARROW_FILESYSTEM=ON \
-DARROW_WITH_LZ4=ON \
-DARROW_WITH_ZSTD=ON \
-DARROW_WITH_SNAPPY=ON \
-DARROW_JSON=ON \
-DARROW_CSV=ON \
-DARROW_DATASET=ON \
-DPROTOBUF_PROTOC_EXECUTABLE=/usr/bin/protoc \
-DARROW_DEPENDENCY_SOURCE=BUNDLED \
.. && \
make -j$(nproc) && \
make install && \
cd ../../python && \
export PYARROW_PARALLEL=4 && \
export ARROW_BUILD_TYPE=release && \
uv pip install -r requirements-build.txt && \
python setup.py build_ext --build-type=$ARROW_BUILD_TYPE --bundle-arrow-cpp bdist_wheel
FROM python-install AS rust
ENV CARGO_HOME=/root/.cargo
ENV RUSTUP_HOME=/root/.rustup
@@ -121,60 +90,18 @@ RUN --mount=type=cache,target=/root/.cache/uv \
FROM python-install AS numba-builder
ARG MAX_JOBS
ARG NUMBA_VERSION=0.61.2
ARG NUMBA_VERSION=0.65.0
WORKDIR /tmp
# Clone all required dependencies
RUN --mount=type=cache,target=/root/.cache/uv \
microdnf install ninja-build gcc gcc-c++ -y && \
git clone --recursive https://github.com/llvm/llvm-project.git -b llvmorg-15.0.7 && \
git clone --recursive https://github.com/numba/llvmlite.git -b v0.44.0 && \
git clone --recursive https://github.com/numba/llvmlite.git -b v0.47.0 && \
git clone --recursive https://github.com/numba/numba.git -b ${NUMBA_VERSION} && \
cd llvm-project && mkdir build && cd build && \
cd llvmlite && \
uv pip install 'cmake<4' 'setuptools<70' numpy && \
export PREFIX=/usr/local && CMAKE_ARGS="${CMAKE_ARGS} -DLLVM_ENABLE_PROJECTS=lld;libunwind;compiler-rt" \
CFLAGS="$(echo $CFLAGS | sed 's/-fno-plt //g')" \
CXXFLAGS="$(echo $CXXFLAGS | sed 's/-fno-plt //g')" \
CMAKE_ARGS="${CMAKE_ARGS} -DFFI_INCLUDE_DIR=$PREFIX/include" \
CMAKE_ARGS="${CMAKE_ARGS} -DFFI_LIBRARY_DIR=$PREFIX/lib" \
cmake -DCMAKE_INSTALL_PREFIX="${PREFIX}" \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_LIBRARY_PATH="${PREFIX}" \
-DLLVM_ENABLE_LIBEDIT=OFF \
-DLLVM_ENABLE_LIBXML2=OFF \
-DLLVM_ENABLE_RTTI=ON \
-DLLVM_ENABLE_TERMINFO=OFF \
-DLLVM_INCLUDE_BENCHMARKS=OFF \
-DLLVM_INCLUDE_DOCS=OFF \
-DLLVM_INCLUDE_EXAMPLES=OFF \
-DLLVM_INCLUDE_GO_TESTS=OFF \
-DLLVM_INCLUDE_TESTS=OFF \
-DLLVM_INCLUDE_UTILS=ON \
-DLLVM_INSTALL_UTILS=ON \
-DLLVM_UTILS_INSTALL_DIR=libexec/llvm \
-DLLVM_BUILD_LLVM_DYLIB=OFF \
-DLLVM_LINK_LLVM_DYLIB=OFF \
-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=WebAssembly \
-DLLVM_ENABLE_FFI=ON \
-DLLVM_ENABLE_Z3_SOLVER=OFF \
-DLLVM_OPTIMIZED_TABLEGEN=ON \
-DCMAKE_POLICY_DEFAULT_CMP0111=NEW \
-DCOMPILER_RT_BUILD_BUILTINS=ON \
-DCOMPILER_RT_BUILTINS_HIDE_SYMBOLS=OFF \
-DCOMPILER_RT_BUILD_LIBFUZZER=OFF \
-DCOMPILER_RT_BUILD_CRT=OFF \
-DCOMPILER_RT_BUILD_MEMPROF=OFF \
-DCOMPILER_RT_BUILD_PROFILE=OFF \
-DCOMPILER_RT_BUILD_SANITIZERS=OFF \
-DCOMPILER_RT_BUILD_XRAY=OFF \
-DCOMPILER_RT_BUILD_GWP_ASAN=OFF \
-DCOMPILER_RT_BUILD_ORC=OFF \
-DCOMPILER_RT_INCLUDE_TESTS=OFF \
${CMAKE_ARGS} -GNinja ../llvm \
&& ninja install . && \
# build llvmlite
cd ../../llvmlite && python setup.py bdist_wheel && \
python setup.py bdist_wheel && \
cd ../numba && \
if ! grep '#include "dynamic_annotations.h"' numba/_dispatcher.cpp; then \
sed -i '/#include "internal\/pycore_atomic.h"/i\#include "dynamic_annotations.h"' numba/_dispatcher.cpp; \
@@ -192,21 +119,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
cd opencv-python && \
python -m build --wheel --installer=uv --outdir /tmp/opencv-python/dist
## Todo(r3hankhan123): Remove guidance-builder stage once vLLM upgrades to new version of llguidance that fixes s390x issues. See https://github.com/guidance-ai/llguidance/issues/330
FROM python-install AS guidance-builder
WORKDIR /tmp
ENV CARGO_HOME=/root/.cargo
ENV RUSTUP_HOME=/root/.rustup
ENV PATH="$CARGO_HOME/bin:$RUSTUP_HOME/bin:$PATH"
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,from=rust,source=/root/.cargo,target=/root/.cargo,rw \
--mount=type=bind,from=rust,source=/root/.rustup,target=/root/.rustup,rw \
git clone https://github.com/guidance-ai/llguidance.git && \
cd llguidance && \
git checkout s390x-fix-v2 && \
uv pip install maturin && \
python -m maturin build --release --out dist --compatibility linux
# # Final build stage
FROM python-install AS vllm-cpu
ARG PYTHON_VERSION
@@ -235,20 +147,16 @@ RUN --mount=type=bind,from=numa-build,src=/tmp/numactl-2.0.19,target=/numactl \
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=bind,from=rust,source=/root/.cargo,target=/root/.cargo,rw \
--mount=type=bind,from=rust,source=/root/.rustup,target=/root/.rustup,rw \
--mount=type=bind,from=pyarrow,source=/tmp/arrow/python/dist,target=/tmp/arrow-wheels \
--mount=type=bind,from=torch-vision,source=/tmp/vision/dist,target=/tmp/vision-wheels/ \
--mount=type=bind,from=hf-xet-builder,source=/tmp/hf-xet/dist,target=/tmp/hf-xet-wheels/ \
--mount=type=bind,from=numba-builder,source=/tmp/llvmlite/dist,target=/tmp/llvmlite-wheels/ \
--mount=type=bind,from=numba-builder,source=/tmp/numba/dist,target=/tmp/numba-wheels/ \
--mount=type=bind,from=opencv-builder,source=/tmp/opencv-python/dist,target=/tmp/opencv-wheels/ \
--mount=type=bind,from=guidance-builder,source=/tmp/llguidance/dist,target=/tmp/guidance-wheels/ \
ARROW_WHL_FILE=$(ls /tmp/arrow-wheels/*.whl) && \
VISION_WHL_FILE=$(ls /tmp/vision-wheels/*.whl) && \
HF_XET_WHL_FILE=$(ls /tmp/hf-xet-wheels/*.whl) && \
LLVM_WHL_FILE=$(ls /tmp/llvmlite-wheels/*.whl) && \
NUMBA_WHL_FILE=$(ls /tmp/numba-wheels/*.whl) && \
OPENCV_WHL_FILE=$(ls /tmp/opencv-wheels/*.whl) && \
GUIDANCE_WHL_FILE=$(ls /tmp/guidance-wheels/*.whl) && \
uv pip install -v \
$ARROW_WHL_FILE \
$VISION_WHL_FILE \
@@ -256,7 +164,6 @@ RUN --mount=type=cache,target=/root/.cache/uv \
$LLVM_WHL_FILE \
$NUMBA_WHL_FILE \
$OPENCV_WHL_FILE \
$GUIDANCE_WHL_FILE \
--torch-backend cpu \
--index-strategy unsafe-best-match \
-r requirements/build/cpu.txt \
+1 -1
View File
@@ -21,7 +21,7 @@ pillow # Required for image processing
prometheus-fastapi-instrumentator >= 8.0.0 # v8 unblocks starlette >= 1.0
tiktoken >= 0.6.0 # Required for DBRX tokenizer
lm-format-enforcer == 0.11.3
llguidance >= 1.7.0, < 1.8.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64" or platform_machine == "ppc64le"
llguidance >= 1.7.0, < 1.8.0; platform_machine == "x86_64" or platform_machine == "arm64" or platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "s390x"
outlines_core == 0.2.14
# required for outlines backend disk cache
diskcache == 5.6.3
@@ -33,6 +33,7 @@ class CpuCommunicator(DeviceCommunicatorBase):
current_platform.get_cpu_architecture() == CpuArchEnum.X86
or current_platform.get_cpu_architecture() == CpuArchEnum.ARM
or current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC
or current_platform.get_cpu_architecture() == CpuArchEnum.S390X
)
and hasattr(torch.ops._C, "init_shm_manager")
and (unique_name.startswith("tp") or unique_name.startswith("pp"))
+6 -1
View File
@@ -247,7 +247,12 @@ class CpuPlatform(Platform):
if (
platform.system() == "Linux"
and cpu_architecture
in (CpuArchEnum.ARM, CpuArchEnum.POWERPC, CpuArchEnum.X86)
in (
CpuArchEnum.ARM,
CpuArchEnum.POWERPC,
CpuArchEnum.X86,
CpuArchEnum.S390X,
)
and not (
"libomp" in ld_preload_str
or "libgomp" in ld_preload_str
+49 -4
View File
@@ -3,6 +3,7 @@
import json
import os
import platform
import subprocess
import sys
from dataclasses import dataclass
@@ -216,9 +217,14 @@ def _get_cpu_list() -> list[LogicalCPUInfo]:
# For MacOS, no user-level CPU affinity and SMT, return all CPUs
return _synthesize_cpu_list()
lscpu_output = subprocess.check_output(
"lscpu --json --extended=CPU,CORE,NODE --online", shell=True, text=True
)
if platform.machine() == "s390x":
lscpu_output = subprocess.check_output(
"lscpu -J -e=CPU,CORE,NODE,SOCKET,BOOK", shell=True, text=True
)
else:
lscpu_output = subprocess.check_output(
"lscpu --json --extended=CPU,CORE,NODE --online", shell=True, text=True
)
# For platforms without NUMA, map bare `-` node to 0 so non-NUMA
# systems keep the existing behavior from #39781.
@@ -233,11 +239,50 @@ def _get_cpu_list() -> list[LogicalCPUInfo]:
lscpu_output,
)
if platform.machine() == "s390x":
# On s390x, use the best topology level as the abstract CPU group
# key via numa_node. The hierarchy is book > socket > core; prefer
# book, fall back to socket if only one book exists.
lscpu_output = re.sub(r'"book":\s*-\s*(,|\n|\})', r'"book": 0\1', lscpu_output)
lscpu_output = re.sub(
r'"socket":\s*-\s*(,|\n|\})', r'"socket": 0\1', lscpu_output
)
raw_cpus = json.loads(lscpu_output)["cpus"]
book_values = set()
socket_values = set()
for entry in raw_cpus:
book_values.add(LogicalCPUInfo._int(str(entry.get("book", "-1"))))
socket_values.add(LogicalCPUInfo._int(str(entry.get("socket", "-1"))))
# Pick the best partitioning level: book first, then socket
if len(book_values - {-1}) > 1:
group_key = "book"
elif len(socket_values - {-1}) > 1:
group_key = "socket"
else:
group_key = None
if group_key is not None:
logical_book_socket_list: list[LogicalCPUInfo] = []
for entry in raw_cpus:
cpu_id = LogicalCPUInfo._int(str(entry.get("cpu", "-1")))
core = LogicalCPUInfo._int(str(entry.get("core", "-1")))
group = LogicalCPUInfo._int(str(entry.get(group_key, "-1")))
if -1 not in (cpu_id, core, group):
logical_book_socket_list.append(
LogicalCPUInfo(id=cpu_id, physical_core=core, numa_node=group)
)
if not logical_book_socket_list:
return _synthesize_cpu_list()
return logical_book_socket_list
logical_cpu_list: list[LogicalCPUInfo] = json.loads(
lscpu_output, object_hook=LogicalCPUInfo.json_decoder
)["cpus"]
# Filter CPUs with invalid attributes
# Filter CPUs with invalid attributes (only require id, core, node)
logical_cpu_list = [
x for x in logical_cpu_list if -1 not in (x.id, x.physical_core, x.numa_node)
]
+8
View File
@@ -204,6 +204,14 @@ class OMPProcessManager:
logical_cpu_list = cr_utils.get_allowed_cpu_list()
local_world_size = self.local_world_size
# On s390x, numa_node is remapped to book IDs (the best CPU
# partitioning level). Derive topology domains from the CPU list
# since they no longer correspond to physical memory nodes.
cpu_arch = current_platform.get_cpu_architecture()
if cpu_arch == CpuArchEnum.S390X:
allowed_numa_nodes = sorted(set(cpu.numa_node for cpu in logical_cpu_list))
assert (
len(allowed_numa_nodes) >= local_world_size or self.simulate_multi_node
), (
+15 -2
View File
@@ -55,9 +55,22 @@ class CPUWorker(Worker):
allowed_memory_nodes,
)
torch.ops._C.init_cpu_memory_env([cpu_core.numa_node])
# On s390x, numa_node may be a synthetic book ID that doesn't
# correspond to a real memory node. Fall back to first visible node.
if cpu_core.numa_node in allowed_memory_nodes:
memory_node = cpu_core.numa_node
else:
logger.warning(
"CPU group key %s is not a valid memory node. "
"Falling back to memory node %s.",
cpu_core.numa_node,
allowed_memory_nodes[0],
)
memory_node = allowed_memory_nodes[0]
memory_status = get_memory_node_info(cpu_core.numa_node)
torch.ops._C.init_cpu_memory_env([memory_node])
memory_status = get_memory_node_info(memory_node)
memory_fraction = vllm_config.cache_config.gpu_memory_utilization
self.requested_cpu_memory = math.ceil(
memory_status.total_memory * memory_fraction