[Build] Remove Ubuntu build-stage option from the CUDA dockerfile (#51060)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: Shengqi Chen <harry-chen@outlook.com>
This commit is contained in:
Michael Goin
2026-08-05 10:50:58 -07:00
committed by GitHub
co-authored by Shengqi Chen
parent 8543522ca7
commit e8586b2878
8 changed files with 42 additions and 97 deletions
@@ -26,6 +26,7 @@ else
--platform linux/arm64 \
--build-arg max_jobs=16 \
--build-arg nvcc_threads=4 \
--build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.0 \
--build-arg torch_cuda_arch_list="9.0 10.0 11.0 12.0" \
--build-arg USE_SCCACHE=1 \
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
@@ -48,7 +48,7 @@ echo "Image not found, proceeding with build..."
# --- CUDA 13.0 for nightly builds ---
# Nightly CI uses CUDA 13.0 while regular CI stays on CUDA 12.9
NIGHTLY_CUDA_VERSION="13.0.2"
NIGHTLY_BUILD_BASE_IMAGE="nvidia/cuda:${NIGHTLY_CUDA_VERSION}-devel-ubuntu22.04"
NIGHTLY_BUILD_BASE_IMAGE="pytorch/manylinux2_28-builder:cuda13.0"
NIGHTLY_FINAL_BASE_IMAGE="nvidia/cuda:${NIGHTLY_CUDA_VERSION}-base-ubuntu22.04"
echo "--- :docker: Building torch nightly image (CUDA ${NIGHTLY_CUDA_VERSION})"
+12 -8
View File
@@ -40,7 +40,7 @@ steps:
agents:
queue: arm64_cpu_queue_release
commands:
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg BUILD_OS=manylinux --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.0 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.0 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
- "mkdir artifacts"
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
- "bash .buildkite/scripts/upload-nightly-wheels.sh"
@@ -54,7 +54,7 @@ steps:
agents:
queue: cpu_queue_release
commands:
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg BUILD_OS=manylinux --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.0 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.0 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
- "mkdir artifacts"
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
- "bash .buildkite/scripts/upload-nightly-wheels.sh"
@@ -78,7 +78,7 @@ steps:
agents:
queue: arm64_cpu_queue_release
commands:
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --build-arg BUILD_OS=manylinux --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda12.9 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda12.9 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
- "mkdir artifacts"
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
- "bash .buildkite/scripts/upload-nightly-wheels.sh"
@@ -149,7 +149,7 @@ steps:
agents:
queue: cpu_queue_release
commands:
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86_CU129}\" --build-arg BUILD_OS=manylinux --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda12.9 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
- "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86_CU129}\" --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda12.9 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
- "mkdir artifacts"
- "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
- "bash .buildkite/scripts/upload-nightly-wheels.sh"
@@ -221,7 +221,7 @@ steps:
--build-arg INSTALL_KV_CONNECTORS=true \
--build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_35}" \
--build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \
--build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 \
--build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.0 \
--target vllm-openai \
--progress plain \
-f docker/Dockerfile .
@@ -249,7 +249,7 @@ steps:
--build-arg INSTALL_KV_CONNECTORS=true \
--build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_35}" \
--build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \
--build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 \
--build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.0 \
--target vllm-openai \
--progress plain \
-f docker/Dockerfile .
@@ -270,6 +270,7 @@ steps:
--build-arg USE_SCCACHE=1 \
--build-arg GIT_REPO_CHECK=1 \
--build-arg CUDA_VERSION=12.9.1 \
--build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda12.9 \
--build-arg torch_cuda_arch_list="${CUDA_ARCH_X86_CU129}" \
--build-arg INSTALL_KV_CONNECTORS=true \
--build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_35}" \
@@ -297,6 +298,7 @@ steps:
--build-arg USE_SCCACHE=1 \
--build-arg GIT_REPO_CHECK=1 \
--build-arg CUDA_VERSION=12.9.1 \
--build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda12.9 \
--build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64_CU129}" \
--build-arg INSTALL_KV_CONNECTORS=true \
--build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_35}" \
@@ -327,7 +329,7 @@ steps:
--build-arg INSTALL_KV_CONNECTORS=true \
--build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_39}" \
--build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \
--build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu24.04 \
--build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.0 \
--target vllm-openai \
--progress plain \
-f docker/Dockerfile .
@@ -356,7 +358,7 @@ steps:
--build-arg INSTALL_KV_CONNECTORS=true \
--build-arg MOONCAKE_WHEEL_AARCH64="${MOONCAKE_WHEEL_AARCH64_2_39}" \
--build-arg MOONCAKE_WHEEL_X86_64="${MOONCAKE_WHEEL_X86_64}" \
--build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu24.04 \
--build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.0 \
--target vllm-openai \
--progress plain \
-f docker/Dockerfile .
@@ -377,6 +379,7 @@ steps:
--build-arg USE_SCCACHE=1 \
--build-arg GIT_REPO_CHECK=1 \
--build-arg CUDA_VERSION=12.9.1 \
--build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda12.9 \
--build-arg UBUNTU_VERSION=24.04 \
--build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \
--build-arg torch_cuda_arch_list="${CUDA_ARCH_X86_CU129}" \
@@ -405,6 +408,7 @@ steps:
--build-arg USE_SCCACHE=1 \
--build-arg GIT_REPO_CHECK=1 \
--build-arg CUDA_VERSION=12.9.1 \
--build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda12.9 \
--build-arg UBUNTU_VERSION=24.04 \
--build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \
--build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64_CU129}" \
@@ -15,6 +15,7 @@ DOCKER_BUILDKIT=1 docker build . \
-t gh200-test \
--build-arg max_jobs=66 \
--build-arg nvcc_threads=2 \
--build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.0 \
--build-arg torch_cuda_arch_list="9.0+PTX"
# Setup cleanup
+24 -83
View File
@@ -35,23 +35,13 @@ ARG NCCL_VERSION=2.30.7
# private registries that use a different repository naming conventions.
#
# Example:
# docker build --build-arg BUILD_BASE_IMAGE=registry.acme.org/mirror/nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04
# docker build --build-arg BUILD_BASE_IMAGE=registry.acme.org/mirror/pytorch/manylinux2_28-builder:cuda13.0
# Important: We build with an old version of Ubuntu to maintain broad
# compatibility with other Linux OSes. The main reason for this is that the
# glibc version is baked into the distro, and binaries built with one glibc
# version are not backwards compatible with OSes that use an earlier version.
ARG BUILD_BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu22.04
# Build wheels against the same glibc floor as PyTorch's published wheels.
ARG BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.0
# Using cuda base image with minimal dependencies necessary for JIT compilation (FlashInfer, DeepGEMM, EP kernels)
ARG FINAL_BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-base-ubuntu${UBUNTU_VERSION}
# OS family of BUILD_BASE_IMAGE. Controls package manager (apt vs dnf) and
# Python bootstrap. Set to "manylinux" alongside a manylinux build base such
# as pytorch/manylinux2_28-builder:cuda13.0 to produce wheels with a glibc
# 2.28 floor (matches PyTorch's own published wheels). Default stays on
# Ubuntu for backwards compatibility.
ARG BUILD_OS=ubuntu
# By parameterizing the Deadsnakes repository URL, we allow third-party to use
# their own mirror. When doing so, we don't benefit from the transparent
# installation of the GPG key of the PPA, as done by add-apt-repository, so we
@@ -106,7 +96,6 @@ FROM ${BUILD_BASE_IMAGE} AS base
ARG TARGETPLATFORM
ARG CUDA_VERSION
ARG PYTHON_VERSION
ARG BUILD_OS
ARG USE_SCCACHE
ARG SCCACHE_DOWNLOAD_URL
ARG SCCACHE_ENDPOINT
@@ -114,8 +103,6 @@ ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
ARG SCCACHE_REGION_NAME=us-west-2
ARG SCCACHE_S3_NO_CREDENTIALS=0
ENV DEBIAN_FRONTEND=noninteractive
# Environment for uv
# Declared BEFORE the installer + `uv venv` invocations below so the uv
# binary, managed Python, download cache, and /opt/venv all land under
@@ -133,45 +120,16 @@ ENV UV_INSTALL_DIR=/opt/uv/bin
ENV PATH="/opt/venv/bin:/opt/uv/bin:$PATH"
ENV VIRTUAL_ENV="/opt/venv"
# Install system dependencies including build tools.
# The Ubuntu path uses apt + deadsnakes-via-uv for Python; the manylinux path
# (AlmaLinux 8, e.g. pytorch/manylinux2_28-builder) uses dnf and the Python
# interpreters pre-installed at /opt/python/cpXY-cpXY/.
RUN if [ "${BUILD_OS}" = "manylinux" ]; then \
# rdma-core-devel provides libibverbs headers; ccache lives in EPEL,
# which the pytorch manylinux image already enables. git/curl/sudo
# are typically pre-installed but listed defensively.
dnf install -y --setopt=install_weak_deps=False \
ccache \
git \
curl \
sudo \
rdma-core-devel \
&& dnf clean all \
&& rm -rf /var/cache/dnf; \
else \
apt-get update -y \
&& apt-get install -y --no-install-recommends \
ccache \
software-properties-common \
git \
curl \
sudo \
python3-pip \
libibverbs-dev \
# GCC 10 was previously pinned to suppress spurious -Wredundant-move warnings
# from CUTLASS (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=92519). That bug
# was fixed in GCC 11. GCC >= 11.3 is now required because PyTorch's C++20 headers
# (pytorch/pytorch#167929) are not compatible with GCC < 11.3.
gcc-11 \
g++-11 \
&& update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 110 --slave /usr/bin/g++ g++ /usr/bin/g++-11 \
# Install python dev headers if available (needed for cmake FindPython on Ubuntu 24.04
# which ships cmake 3.28 and requires Development.SABIModule; silently skipped on
# Ubuntu 20.04/22.04 where python3.x-dev is not available without a PPA)
&& (apt-get install -y --no-install-recommends python${PYTHON_VERSION}-dev 2>/dev/null || true) \
&& rm -rf /var/lib/apt/lists/*; \
fi
# Install system dependencies including build tools. The PyTorch manylinux
# image enables EPEL, which provides ccache.
RUN dnf install -y --setopt=install_weak_deps=False \
ccache \
git \
curl \
sudo \
rdma-core-devel \
&& dnf clean all \
&& rm -rf /var/cache/dnf
# Install sccache once in base so Rust and CMake/CUDA build stages share the
# same binary and remote cache configuration.
@@ -194,8 +152,8 @@ ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}}
ENV SCCACHE_S3_NO_CREDENTIALS=${USE_SCCACHE:+${SCCACHE_S3_NO_CREDENTIALS}}
ENV SCCACHE_IDLE_TIMEOUT=${USE_SCCACHE:+0}
# Install uv and bootstrap /opt/venv. Both paths converge on /opt/venv so all
# downstream stages stay distro-agnostic.
# Install uv and bootstrap /opt/venv from the Python interpreter included in
# the manylinux image.
RUN mkdir -p "${UV_PYTHON_INSTALL_DIR}" "${UV_CACHE_DIR}" "${UV_INSTALL_DIR}" \
&& chmod -R a+rX /opt/uv \
&& curl -LsSf https://astral.sh/uv/install.sh | sh \
@@ -203,15 +161,9 @@ RUN mkdir -p "${UV_PYTHON_INSTALL_DIR}" "${UV_CACHE_DIR}" "${UV_INSTALL_DIR}" \
# pip` works regardless of how uv happens to link the venv back to the
# managed Python install (which, at a non-default UV_PYTHON_INSTALL_DIR,
# doesn't always expose ensurepip via the default venv layout).
&& if [ "${BUILD_OS}" = "manylinux" ]; then \
# manylinux images ship Python at /opt/python/cpXY-cpXY/; point uv
# at the matching interpreter rather than letting it fetch one.
PYV_NODOT=$(echo ${PYTHON_VERSION} | tr -d '.') \
&& MANYLINUX_PY=/opt/python/cp${PYV_NODOT}-cp${PYV_NODOT}/bin/python${PYTHON_VERSION} \
&& uv venv --seed /opt/venv --python "$MANYLINUX_PY"; \
else \
uv venv --seed /opt/venv --python ${PYTHON_VERSION}; \
fi \
&& PYV_NODOT=$(echo ${PYTHON_VERSION} | tr -d '.') \
&& MANYLINUX_PY=/opt/python/cp${PYV_NODOT}-cp${PYV_NODOT}/bin/python${PYTHON_VERSION} \
&& uv venv --seed /opt/venv --python "$MANYLINUX_PY" \
&& rm -f /usr/bin/python3 /usr/bin/python3-config /usr/bin/pip \
&& ln -sf /opt/venv/bin/python3 /usr/bin/python3 \
&& ln -sf /opt/venv/bin/python3-config /usr/bin/python3-config \
@@ -295,21 +247,13 @@ ENV TORCH_CUDA_ARCH_LIST=${torch_cuda_arch_list}
# This stage reuses the Python environment from base and runs in parallel with
# csrc-build/extensions-build.
FROM base AS rust-build
ARG BUILD_OS
ARG USE_SCCACHE
ARG SCCACHE_ENDPOINT
# Install native tools needed only for Rust/protoc builds.
RUN if [ "${BUILD_OS}" = "manylinux" ]; then \
dnf install -y --setopt=install_weak_deps=False \
make unzip \
&& dnf clean all && rm -rf /var/cache/dnf; \
else \
apt-get update -y \
&& apt-get install -y --no-install-recommends \
make unzip \
&& rm -rf /var/lib/apt/lists/*; \
fi
RUN dnf install -y --setopt=install_weak_deps=False make unzip \
&& dnf clean all \
&& rm -rf /var/cache/dnf
COPY tools/install_protoc.sh /tmp/install_protoc.sh
RUN /tmp/install_protoc.sh && rm /tmp/install_protoc.sh
@@ -606,7 +550,6 @@ FROM base AS dev
ARG PIP_INDEX_URL UV_INDEX_URL
ARG PIP_EXTRA_INDEX_URL UV_EXTRA_INDEX_URL
ARG PYTORCH_CUDA_INDEX_BASE_URL
ARG BUILD_OS
# This timeout (in seconds) is necessary when installing some dependencies via uv since it's likely to time out
# Reference: https://github.com/astral-sh/uv/pull/1694
@@ -616,11 +559,9 @@ ENV UV_INDEX_STRATEGY="unsafe-best-match"
ENV UV_LINK_MODE=copy
# Install libnuma-dev, required by fastsafetensors (fixes #20384)
RUN if [ "${BUILD_OS}" = "manylinux" ]; then \
dnf install -y numactl-devel && dnf clean all && rm -rf /var/cache/dnf; \
else \
apt-get update && apt-get install -y --no-install-recommends libnuma-dev && rm -rf /var/lib/apt/lists/*; \
fi
RUN dnf install -y numactl-devel \
&& dnf clean all \
&& rm -rf /var/cache/dnf
# We can specify the standard or nightly build of PyTorch
+1 -4
View File
@@ -14,14 +14,11 @@
"default": "2.30.7"
},
"BUILD_BASE_IMAGE": {
"default": "nvidia/cuda:13.0.3-devel-ubuntu22.04"
"default": "pytorch/manylinux2_28-builder:cuda13.0"
},
"FINAL_BASE_IMAGE": {
"default": "nvidia/cuda:13.0.3-base-ubuntu22.04"
},
"BUILD_OS": {
"default": "ubuntu"
},
"GET_PIP_URL": {
"default": "https://bootstrap.pypa.io/get-pip.py"
},
Binary file not shown.

Before

Width:  |  Height:  |  Size: 395 KiB

After

Width:  |  Height:  |  Size: 398 KiB

@@ -384,6 +384,7 @@ A docker container can be built for aarch64 systems such as the Nvidia Grace-Hop
-t vllm/vllm-gh200-openai:latest \
--build-arg max_jobs=66 \
--build-arg nvcc_threads=2 \
--build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.0 \
--build-arg torch_cuda_arch_list="9.0 10.0+PTX"
```
@@ -394,7 +395,7 @@ For (G)B300, we recommend using CUDA 13, as shown in the following command.
```bash
DOCKER_BUILDKIT=1 docker build \
--build-arg CUDA_VERSION=13.0.2 \
--build-arg BUILD_BASE_IMAGE=nvidia/cuda:13.0.2-devel-ubuntu22.04 \
--build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.0 \
--build-arg max_jobs=256 \
--build-arg nvcc_threads=2 \
--build-arg torch_cuda_arch_list='9.0 10.0+PTX' \