[CI] Build CPU image against torch nightly for TORCH_NIGHTLY runs (#48750)

Signed-off-by: Andrey Talman <atalman@users.noreply.github.com>
Signed-off-by: Andrey Talman <atalman@fb.com>
Co-authored-by: Andrey Talman <atalman@users.noreply.github.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
Andrey Talman
2026-08-31 17:25:57 -07:00
committed by GitHub
co-authored by Andrey Talman Claude Kevin H. Luu
parent e29af0a2af
commit 8600db5dff
4 changed files with 104 additions and 13 deletions
+12 -1
View File
@@ -9,7 +9,17 @@ fi
REGISTRY=$1
REPO=$2
BUILDKITE_COMMIT=$3
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64"
# When TORCH_NIGHTLY=1, build against torch nightly and tag it with the
# -torch-nightly-arm64 suffix the DGX Spark test steps pull on the nightly lane
# (ci-infra get_image(arm64=True) appends -torch-nightly before -arm64).
PYTORCH_NIGHTLY_ARGS=()
if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-torch-nightly-arm64"
PYTORCH_NIGHTLY_ARGS=(--build-arg PYTORCH_NIGHTLY=1)
else
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64"
fi
# authenticate with AWS ECR
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
@@ -30,6 +40,7 @@ else
--build-arg torch_cuda_arch_list="9.0 10.0 11.0 12.0" \
--build-arg USE_SCCACHE=1 \
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
"${PYTORCH_NIGHTLY_ARGS[@]}" \
--tag "$IMAGE" \
--target test \
--progress plain .
+25 -8
View File
@@ -9,7 +9,16 @@ fi
REGISTRY=$1
REPO=$2
BUILDKITE_COMMIT=$3
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu"
# When TORCH_NIGHTLY=1, build the CPU image against torch nightly and tag it
# with the -torch-nightly-cpu suffix the test steps pull on the nightly lane.
PYTORCH_NIGHTLY_ARGS=()
if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-torch-nightly-cpu"
PYTORCH_NIGHTLY_ARGS=(--build-arg PYTORCH_NIGHTLY=1)
else
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu"
fi
# replace invalid characters in Docker image tags and truncate to 128 chars
clean_docker_tag() {
@@ -20,32 +29,39 @@ clean_docker_tag() {
# resolve and set: CACHE_TO, CACHE_FROM, CACHE_FROM_BASE_BRANCH, CACHE_FROM_MAIN
# Reuses the same ECR cache repos as the CUDA image build, with an
# "x86_cpu" suffix on every tag so CPU and CUDA cache blobs stay isolated.
# The nightly lane appends "-nightly": it resolves a different torch, so
# sharing a cache ref with the regular lane would have the two evict each
# other from the same ref on every run.
prepare_cache_tags() {
TEST_CACHE_ECR="936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-test-cache"
MAIN_CACHE_ECR="936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-postmerge-cache"
CACHE_SUFFIX="x86_cpu"
if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then
CACHE_SUFFIX="x86_cpu-nightly"
fi
if [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]]; then
if [[ "${BUILDKITE_BRANCH:-}" == "main" ]]; then
cache="${MAIN_CACHE_ECR}:latest-x86_cpu"
cache="${MAIN_CACHE_ECR}:latest-${CACHE_SUFFIX}"
else
clean_branch=$(clean_docker_tag "${BUILDKITE_BRANCH:-unknown}")
cache="${TEST_CACHE_ECR}:${clean_branch}-x86_cpu"
cache="${TEST_CACHE_ECR}:${clean_branch}-${CACHE_SUFFIX}"
fi
CACHE_TO="$cache"
CACHE_FROM="$cache"
CACHE_FROM_BASE_BRANCH="$cache"
else
CACHE_TO="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}-x86_cpu"
CACHE_FROM="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}-x86_cpu"
CACHE_TO="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}-${CACHE_SUFFIX}"
CACHE_FROM="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}-${CACHE_SUFFIX}"
if [[ "${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" == "main" ]]; then
CACHE_FROM_BASE_BRANCH="${MAIN_CACHE_ECR}:latest-x86_cpu"
CACHE_FROM_BASE_BRANCH="${MAIN_CACHE_ECR}:latest-${CACHE_SUFFIX}"
else
clean_base=$(clean_docker_tag "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}")
CACHE_FROM_BASE_BRANCH="${TEST_CACHE_ECR}:${clean_base}-x86_cpu"
CACHE_FROM_BASE_BRANCH="${TEST_CACHE_ECR}:${clean_base}-${CACHE_SUFFIX}"
fi
fi
CACHE_FROM_MAIN="${MAIN_CACHE_ECR}:latest-x86_cpu"
CACHE_FROM_MAIN="${MAIN_CACHE_ECR}:latest-${CACHE_SUFFIX}"
}
# authenticate with AWS ECR (public, for the image; private, for the cache repos)
@@ -90,6 +106,7 @@ else
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
--build-arg VLLM_CPU_X86=true \
--build-arg USE_SCCACHE=1 \
"${PYTORCH_NIGHTLY_ARGS[@]}" \
--tag "$IMAGE" \
--target vllm-test \
"${CACHE_FROM_ARGS[@]}" \
+65 -4
View File
@@ -100,13 +100,33 @@ ENV UV_HTTP_TIMEOUT=500
ENV UV_INDEX_STRATEGY="unsafe-best-match"
ENV UV_LINK_MODE="copy"
# PyTorch provides its own indexes for standard and nightly builds.
# PYTORCH_NIGHTLY=1 builds the CPU image against torch nightly (parity with the
# GPU torch-nightly lane); default 0 keeps the pinned release CPU build.
ARG PYTORCH_NIGHTLY=0
ARG PYTORCH_CPU_INDEX_BASE_URL=https://download.pytorch.org/whl
# Copy requirements files for installation
COPY requirements/common.txt requirements/common.txt
COPY requirements/cpu.txt requirements/cpu.txt
COPY use_existing_torch.py use_existing_torch.py
# For nightly: install torch nightly (CPU) first, strip torch pins from the
# requirements via use_existing_torch.py so they can't pull it back to a release
# build, then install the rest from the nightly CPU index (no --torch-backend,
# which would force the stable channel). Release path is unchanged.
RUN --mount=type=cache,target=/root/.cache/uv \
uv pip install --upgrade pip && \
uv pip install -r requirements/cpu.txt --torch-backend cpu
if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
echo "Installing torch nightly (CPU)..." \
&& uv pip install torch torchaudio torchvision torchcodec --pre \
--index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu \
&& python3 use_existing_torch.py --prefix \
&& uv pip install -r requirements/cpu.txt \
--extra-index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu; \
else \
uv pip install -r requirements/cpu.txt --torch-backend cpu; \
fi
ENV TARGETARCH=${TARGETARCH}
@@ -191,6 +211,10 @@ RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \
# build it never uses.
FROM base-arch AS vllm-src
# Re-declare torch-nightly args (ARGs do not cross FROM)
ARG PYTORCH_NIGHTLY=0
ARG PYTORCH_CPU_INDEX_BASE_URL=https://download.pytorch.org/whl
ARG GIT_REPO_CHECK=0
# Support for cross-compilation with x86 ISA including AVX2 and AVX512: docker build --build-arg VLLM_CPU_X86="true" ...
ARG VLLM_CPU_X86=0
@@ -215,7 +239,13 @@ RUN if [ "$TARGETARCH" = "arm64" ] && [ "$VLLM_CPU_X86" != "0" ]; then \
COPY requirements/build/cpu.txt requirements/build/cpu.txt
RUN --mount=type=cache,target=/root/.cache/uv \
uv pip install -r requirements/build/cpu.txt --torch-backend cpu
if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
python3 /workspace/use_existing_torch.py --prefix \
&& uv pip install -r requirements/build/cpu.txt \
--extra-index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu; \
else \
uv pip install -r requirements/build/cpu.txt --torch-backend cpu; \
fi
COPY . .
@@ -226,6 +256,18 @@ COPY --from=rust-build /workspace/vllm/_rust_*.so vllm/
RUN if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
# For torch nightly, strip the torch/vision/audio/codec pins from the source
# tree's requirements *after* `COPY . .` (which restored the pinned files) so
# the wheel built below does not hard-pin the release torch (e.g.
# torch==2.11.0+cpu). setup.py reads requirements/cpu.txt into install_requires;
# leaving the pin makes `uv pip install dist/*.whl` unsatisfiable against the
# nightly index. The nightly torch installed in the base image satisfies the
# now-unpinned dependency. Done at the end of vllm-src so vllm-build and
# vllm-dev, which both derive from it, inherit the unpinned tree.
RUN if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
python3 /workspace/use_existing_torch.py --prefix; \
fi
######################### BUILD IMAGE #########################
FROM vllm-src AS vllm-build
@@ -283,6 +325,10 @@ RUN --mount=type=cache,target=/root/.cache/uv \
######################### TEST DEPS #########################
FROM base-arch AS vllm-test-deps
# Re-declare torch-nightly args (ARGs do not cross FROM)
ARG PYTORCH_NIGHTLY=0
ARG PYTORCH_CPU_INDEX_BASE_URL=https://download.pytorch.org/whl
WORKDIR /vllm-workspace
# Test requirements are compiled from requirements/test/cuda.in into
@@ -297,11 +343,21 @@ RUN case "$(uname -m)" in \
esac
RUN --mount=type=cache,target=/root/.cache/uv \
uv pip install -r requirements/test/cpu.txt --torch-backend cpu
if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
python3 /workspace/use_existing_torch.py --prefix \
&& uv pip install -r requirements/test/cpu.txt \
--extra-index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu; \
else \
uv pip install -r requirements/test/cpu.txt --torch-backend cpu; \
fi
######################### DEV IMAGE #########################
FROM vllm-src AS vllm-dev
# Re-declare torch-nightly args (ARGs do not cross FROM)
ARG PYTORCH_NIGHTLY=0
ARG PYTORCH_CPU_INDEX_BASE_URL=https://download.pytorch.org/whl
WORKDIR /vllm-workspace
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
@@ -323,7 +379,12 @@ COPY --from=vllm-test-deps /vllm-workspace/requirements/test/cpu.txt requirement
RUN --mount=type=cache,target=/root/.cache/uv \
uv pip install -r requirements/lint.txt && \
uv pip install -r requirements/test/cpu.txt --torch-backend cpu && \
if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
uv pip install -r requirements/test/cpu.txt \
--extra-index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu; \
else \
uv pip install -r requirements/test/cpu.txt --torch-backend cpu; \
fi && \
pre-commit install --hook-type pre-commit --hook-type commit-msg
# Installed after requirements/test/cpu.txt (which pins triton==3.6.0) so the
+2
View File
@@ -11,10 +11,12 @@ TORCH_LIB_PREFIXES = (
"torch=",
"torchvision=",
"torchaudio=",
"torchcodec=",
# pyproject.toml
'"torch =',
'"torchvision =',
'"torchaudio =',
'"torchcodec =',
)