From 8600db5dff18054f7a4314f6f8bba4259e3e2a98 Mon Sep 17 00:00:00 2001 From: Andrey Talman Date: Mon, 31 Aug 2026 20:25:57 -0400 Subject: [PATCH] [CI] Build CPU image against torch nightly for TORCH_NIGHTLY runs (#48750) Signed-off-by: Andrey Talman Signed-off-by: Andrey Talman Co-authored-by: Andrey Talman Co-authored-by: Claude Co-authored-by: Kevin H. Luu --- .buildkite/image_build/image_build_arm64.sh | 13 +++- .buildkite/image_build/image_build_cpu.sh | 33 +++++++--- docker/Dockerfile.cpu | 69 +++++++++++++++++++-- use_existing_torch.py | 2 + 4 files changed, 104 insertions(+), 13 deletions(-) diff --git a/.buildkite/image_build/image_build_arm64.sh b/.buildkite/image_build/image_build_arm64.sh index 4db8a2f546..69d7f55b9f 100755 --- a/.buildkite/image_build/image_build_arm64.sh +++ b/.buildkite/image_build/image_build_arm64.sh @@ -9,7 +9,17 @@ fi REGISTRY=$1 REPO=$2 BUILDKITE_COMMIT=$3 -IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64" + +# When TORCH_NIGHTLY=1, build against torch nightly and tag it with the +# -torch-nightly-arm64 suffix the DGX Spark test steps pull on the nightly lane +# (ci-infra get_image(arm64=True) appends -torch-nightly before -arm64). +PYTORCH_NIGHTLY_ARGS=() +if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then + IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-torch-nightly-arm64" + PYTORCH_NIGHTLY_ARGS=(--build-arg PYTORCH_NIGHTLY=1) +else + IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64" +fi # authenticate with AWS ECR aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true @@ -30,6 +40,7 @@ else --build-arg torch_cuda_arch_list="9.0 10.0 11.0 12.0" \ --build-arg USE_SCCACHE=1 \ --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ + "${PYTORCH_NIGHTLY_ARGS[@]}" \ --tag "$IMAGE" \ --target test \ --progress plain . diff --git a/.buildkite/image_build/image_build_cpu.sh b/.buildkite/image_build/image_build_cpu.sh index 573c4564f1..0f27538965 100755 --- a/.buildkite/image_build/image_build_cpu.sh +++ b/.buildkite/image_build/image_build_cpu.sh @@ -9,7 +9,16 @@ fi REGISTRY=$1 REPO=$2 BUILDKITE_COMMIT=$3 -IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu" + +# When TORCH_NIGHTLY=1, build the CPU image against torch nightly and tag it +# with the -torch-nightly-cpu suffix the test steps pull on the nightly lane. +PYTORCH_NIGHTLY_ARGS=() +if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then + IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-torch-nightly-cpu" + PYTORCH_NIGHTLY_ARGS=(--build-arg PYTORCH_NIGHTLY=1) +else + IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu" +fi # replace invalid characters in Docker image tags and truncate to 128 chars clean_docker_tag() { @@ -20,32 +29,39 @@ clean_docker_tag() { # resolve and set: CACHE_TO, CACHE_FROM, CACHE_FROM_BASE_BRANCH, CACHE_FROM_MAIN # Reuses the same ECR cache repos as the CUDA image build, with an # "x86_cpu" suffix on every tag so CPU and CUDA cache blobs stay isolated. +# The nightly lane appends "-nightly": it resolves a different torch, so +# sharing a cache ref with the regular lane would have the two evict each +# other from the same ref on every run. prepare_cache_tags() { TEST_CACHE_ECR="936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-test-cache" MAIN_CACHE_ECR="936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-postmerge-cache" + CACHE_SUFFIX="x86_cpu" + if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then + CACHE_SUFFIX="x86_cpu-nightly" + fi if [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]]; then if [[ "${BUILDKITE_BRANCH:-}" == "main" ]]; then - cache="${MAIN_CACHE_ECR}:latest-x86_cpu" + cache="${MAIN_CACHE_ECR}:latest-${CACHE_SUFFIX}" else clean_branch=$(clean_docker_tag "${BUILDKITE_BRANCH:-unknown}") - cache="${TEST_CACHE_ECR}:${clean_branch}-x86_cpu" + cache="${TEST_CACHE_ECR}:${clean_branch}-${CACHE_SUFFIX}" fi CACHE_TO="$cache" CACHE_FROM="$cache" CACHE_FROM_BASE_BRANCH="$cache" else - CACHE_TO="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}-x86_cpu" - CACHE_FROM="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}-x86_cpu" + CACHE_TO="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}-${CACHE_SUFFIX}" + CACHE_FROM="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}-${CACHE_SUFFIX}" if [[ "${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" == "main" ]]; then - CACHE_FROM_BASE_BRANCH="${MAIN_CACHE_ECR}:latest-x86_cpu" + CACHE_FROM_BASE_BRANCH="${MAIN_CACHE_ECR}:latest-${CACHE_SUFFIX}" else clean_base=$(clean_docker_tag "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}") - CACHE_FROM_BASE_BRANCH="${TEST_CACHE_ECR}:${clean_base}-x86_cpu" + CACHE_FROM_BASE_BRANCH="${TEST_CACHE_ECR}:${clean_base}-${CACHE_SUFFIX}" fi fi - CACHE_FROM_MAIN="${MAIN_CACHE_ECR}:latest-x86_cpu" + CACHE_FROM_MAIN="${MAIN_CACHE_ECR}:latest-${CACHE_SUFFIX}" } # authenticate with AWS ECR (public, for the image; private, for the cache repos) @@ -90,6 +106,7 @@ else --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ --build-arg VLLM_CPU_X86=true \ --build-arg USE_SCCACHE=1 \ + "${PYTORCH_NIGHTLY_ARGS[@]}" \ --tag "$IMAGE" \ --target vllm-test \ "${CACHE_FROM_ARGS[@]}" \ diff --git a/docker/Dockerfile.cpu b/docker/Dockerfile.cpu index 2c9adf4ace..827ef736b0 100644 --- a/docker/Dockerfile.cpu +++ b/docker/Dockerfile.cpu @@ -100,13 +100,33 @@ ENV UV_HTTP_TIMEOUT=500 ENV UV_INDEX_STRATEGY="unsafe-best-match" ENV UV_LINK_MODE="copy" +# PyTorch provides its own indexes for standard and nightly builds. +# PYTORCH_NIGHTLY=1 builds the CPU image against torch nightly (parity with the +# GPU torch-nightly lane); default 0 keeps the pinned release CPU build. +ARG PYTORCH_NIGHTLY=0 +ARG PYTORCH_CPU_INDEX_BASE_URL=https://download.pytorch.org/whl + # Copy requirements files for installation COPY requirements/common.txt requirements/common.txt COPY requirements/cpu.txt requirements/cpu.txt +COPY use_existing_torch.py use_existing_torch.py +# For nightly: install torch nightly (CPU) first, strip torch pins from the +# requirements via use_existing_torch.py so they can't pull it back to a release +# build, then install the rest from the nightly CPU index (no --torch-backend, +# which would force the stable channel). Release path is unchanged. RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install --upgrade pip && \ - uv pip install -r requirements/cpu.txt --torch-backend cpu + if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ + echo "Installing torch nightly (CPU)..." \ + && uv pip install torch torchaudio torchvision torchcodec --pre \ + --index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu \ + && python3 use_existing_torch.py --prefix \ + && uv pip install -r requirements/cpu.txt \ + --extra-index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu; \ + else \ + uv pip install -r requirements/cpu.txt --torch-backend cpu; \ + fi ENV TARGETARCH=${TARGETARCH} @@ -191,6 +211,10 @@ RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \ # build it never uses. FROM base-arch AS vllm-src +# Re-declare torch-nightly args (ARGs do not cross FROM) +ARG PYTORCH_NIGHTLY=0 +ARG PYTORCH_CPU_INDEX_BASE_URL=https://download.pytorch.org/whl + ARG GIT_REPO_CHECK=0 # Support for cross-compilation with x86 ISA including AVX2 and AVX512: docker build --build-arg VLLM_CPU_X86="true" ... ARG VLLM_CPU_X86=0 @@ -215,7 +239,13 @@ RUN if [ "$TARGETARCH" = "arm64" ] && [ "$VLLM_CPU_X86" != "0" ]; then \ COPY requirements/build/cpu.txt requirements/build/cpu.txt RUN --mount=type=cache,target=/root/.cache/uv \ - uv pip install -r requirements/build/cpu.txt --torch-backend cpu + if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ + python3 /workspace/use_existing_torch.py --prefix \ + && uv pip install -r requirements/build/cpu.txt \ + --extra-index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu; \ + else \ + uv pip install -r requirements/build/cpu.txt --torch-backend cpu; \ + fi COPY . . @@ -226,6 +256,18 @@ COPY --from=rust-build /workspace/vllm/_rust_*.so vllm/ RUN if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi +# For torch nightly, strip the torch/vision/audio/codec pins from the source +# tree's requirements *after* `COPY . .` (which restored the pinned files) so +# the wheel built below does not hard-pin the release torch (e.g. +# torch==2.11.0+cpu). setup.py reads requirements/cpu.txt into install_requires; +# leaving the pin makes `uv pip install dist/*.whl` unsatisfiable against the +# nightly index. The nightly torch installed in the base image satisfies the +# now-unpinned dependency. Done at the end of vllm-src so vllm-build and +# vllm-dev, which both derive from it, inherit the unpinned tree. +RUN if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ + python3 /workspace/use_existing_torch.py --prefix; \ + fi + ######################### BUILD IMAGE ######################### FROM vllm-src AS vllm-build @@ -283,6 +325,10 @@ RUN --mount=type=cache,target=/root/.cache/uv \ ######################### TEST DEPS ######################### FROM base-arch AS vllm-test-deps +# Re-declare torch-nightly args (ARGs do not cross FROM) +ARG PYTORCH_NIGHTLY=0 +ARG PYTORCH_CPU_INDEX_BASE_URL=https://download.pytorch.org/whl + WORKDIR /vllm-workspace # Test requirements are compiled from requirements/test/cuda.in into @@ -297,11 +343,21 @@ RUN case "$(uname -m)" in \ esac RUN --mount=type=cache,target=/root/.cache/uv \ - uv pip install -r requirements/test/cpu.txt --torch-backend cpu + if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ + python3 /workspace/use_existing_torch.py --prefix \ + && uv pip install -r requirements/test/cpu.txt \ + --extra-index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu; \ + else \ + uv pip install -r requirements/test/cpu.txt --torch-backend cpu; \ + fi ######################### DEV IMAGE ######################### FROM vllm-src AS vllm-dev +# Re-declare torch-nightly args (ARGs do not cross FROM) +ARG PYTORCH_NIGHTLY=0 +ARG PYTORCH_CPU_INDEX_BASE_URL=https://download.pytorch.org/whl + WORKDIR /vllm-workspace RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ @@ -323,7 +379,12 @@ COPY --from=vllm-test-deps /vllm-workspace/requirements/test/cpu.txt requirement RUN --mount=type=cache,target=/root/.cache/uv \ uv pip install -r requirements/lint.txt && \ - uv pip install -r requirements/test/cpu.txt --torch-backend cpu && \ + if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \ + uv pip install -r requirements/test/cpu.txt \ + --extra-index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu; \ + else \ + uv pip install -r requirements/test/cpu.txt --torch-backend cpu; \ + fi && \ pre-commit install --hook-type pre-commit --hook-type commit-msg # Installed after requirements/test/cpu.txt (which pins triton==3.6.0) so the diff --git a/use_existing_torch.py b/use_existing_torch.py index d090c4a4c4..8f79bc6c52 100644 --- a/use_existing_torch.py +++ b/use_existing_torch.py @@ -11,10 +11,12 @@ TORCH_LIB_PREFIXES = ( "torch=", "torchvision=", "torchaudio=", + "torchcodec=", # pyproject.toml '"torch =', '"torchvision =', '"torchaudio =', + '"torchcodec =', )