[CI] Build CPU image against torch nightly for TORCH_NIGHTLY runs (#48750)
Signed-off-by: Andrey Talman <atalman@users.noreply.github.com> Signed-off-by: Andrey Talman <atalman@fb.com> Co-authored-by: Andrey Talman <atalman@users.noreply.github.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Andrey Talman
Claude
Kevin H. Luu
parent
e29af0a2af
commit
8600db5dff
@@ -9,7 +9,17 @@ fi
|
||||
REGISTRY=$1
|
||||
REPO=$2
|
||||
BUILDKITE_COMMIT=$3
|
||||
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64"
|
||||
|
||||
# When TORCH_NIGHTLY=1, build against torch nightly and tag it with the
|
||||
# -torch-nightly-arm64 suffix the DGX Spark test steps pull on the nightly lane
|
||||
# (ci-infra get_image(arm64=True) appends -torch-nightly before -arm64).
|
||||
PYTORCH_NIGHTLY_ARGS=()
|
||||
if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then
|
||||
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-torch-nightly-arm64"
|
||||
PYTORCH_NIGHTLY_ARGS=(--build-arg PYTORCH_NIGHTLY=1)
|
||||
else
|
||||
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64"
|
||||
fi
|
||||
|
||||
# authenticate with AWS ECR
|
||||
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true
|
||||
@@ -30,6 +40,7 @@ else
|
||||
--build-arg torch_cuda_arch_list="9.0 10.0 11.0 12.0" \
|
||||
--build-arg USE_SCCACHE=1 \
|
||||
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
|
||||
"${PYTORCH_NIGHTLY_ARGS[@]}" \
|
||||
--tag "$IMAGE" \
|
||||
--target test \
|
||||
--progress plain .
|
||||
|
||||
@@ -9,7 +9,16 @@ fi
|
||||
REGISTRY=$1
|
||||
REPO=$2
|
||||
BUILDKITE_COMMIT=$3
|
||||
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu"
|
||||
|
||||
# When TORCH_NIGHTLY=1, build the CPU image against torch nightly and tag it
|
||||
# with the -torch-nightly-cpu suffix the test steps pull on the nightly lane.
|
||||
PYTORCH_NIGHTLY_ARGS=()
|
||||
if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then
|
||||
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-torch-nightly-cpu"
|
||||
PYTORCH_NIGHTLY_ARGS=(--build-arg PYTORCH_NIGHTLY=1)
|
||||
else
|
||||
IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu"
|
||||
fi
|
||||
|
||||
# replace invalid characters in Docker image tags and truncate to 128 chars
|
||||
clean_docker_tag() {
|
||||
@@ -20,32 +29,39 @@ clean_docker_tag() {
|
||||
# resolve and set: CACHE_TO, CACHE_FROM, CACHE_FROM_BASE_BRANCH, CACHE_FROM_MAIN
|
||||
# Reuses the same ECR cache repos as the CUDA image build, with an
|
||||
# "x86_cpu" suffix on every tag so CPU and CUDA cache blobs stay isolated.
|
||||
# The nightly lane appends "-nightly": it resolves a different torch, so
|
||||
# sharing a cache ref with the regular lane would have the two evict each
|
||||
# other from the same ref on every run.
|
||||
prepare_cache_tags() {
|
||||
TEST_CACHE_ECR="936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-test-cache"
|
||||
MAIN_CACHE_ECR="936637512419.dkr.ecr.us-east-1.amazonaws.com/vllm-ci-postmerge-cache"
|
||||
CACHE_SUFFIX="x86_cpu"
|
||||
if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then
|
||||
CACHE_SUFFIX="x86_cpu-nightly"
|
||||
fi
|
||||
|
||||
if [[ "${BUILDKITE_PULL_REQUEST:-false}" == "false" ]]; then
|
||||
if [[ "${BUILDKITE_BRANCH:-}" == "main" ]]; then
|
||||
cache="${MAIN_CACHE_ECR}:latest-x86_cpu"
|
||||
cache="${MAIN_CACHE_ECR}:latest-${CACHE_SUFFIX}"
|
||||
else
|
||||
clean_branch=$(clean_docker_tag "${BUILDKITE_BRANCH:-unknown}")
|
||||
cache="${TEST_CACHE_ECR}:${clean_branch}-x86_cpu"
|
||||
cache="${TEST_CACHE_ECR}:${clean_branch}-${CACHE_SUFFIX}"
|
||||
fi
|
||||
CACHE_TO="$cache"
|
||||
CACHE_FROM="$cache"
|
||||
CACHE_FROM_BASE_BRANCH="$cache"
|
||||
else
|
||||
CACHE_TO="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}-x86_cpu"
|
||||
CACHE_FROM="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}-x86_cpu"
|
||||
CACHE_TO="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}-${CACHE_SUFFIX}"
|
||||
CACHE_FROM="${TEST_CACHE_ECR}:pr-${BUILDKITE_PULL_REQUEST}-${CACHE_SUFFIX}"
|
||||
if [[ "${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}" == "main" ]]; then
|
||||
CACHE_FROM_BASE_BRANCH="${MAIN_CACHE_ECR}:latest-x86_cpu"
|
||||
CACHE_FROM_BASE_BRANCH="${MAIN_CACHE_ECR}:latest-${CACHE_SUFFIX}"
|
||||
else
|
||||
clean_base=$(clean_docker_tag "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}")
|
||||
CACHE_FROM_BASE_BRANCH="${TEST_CACHE_ECR}:${clean_base}-x86_cpu"
|
||||
CACHE_FROM_BASE_BRANCH="${TEST_CACHE_ECR}:${clean_base}-${CACHE_SUFFIX}"
|
||||
fi
|
||||
fi
|
||||
|
||||
CACHE_FROM_MAIN="${MAIN_CACHE_ECR}:latest-x86_cpu"
|
||||
CACHE_FROM_MAIN="${MAIN_CACHE_ECR}:latest-${CACHE_SUFFIX}"
|
||||
}
|
||||
|
||||
# authenticate with AWS ECR (public, for the image; private, for the cache repos)
|
||||
@@ -90,6 +106,7 @@ else
|
||||
--build-arg buildkite_commit="$BUILDKITE_COMMIT" \
|
||||
--build-arg VLLM_CPU_X86=true \
|
||||
--build-arg USE_SCCACHE=1 \
|
||||
"${PYTORCH_NIGHTLY_ARGS[@]}" \
|
||||
--tag "$IMAGE" \
|
||||
--target vllm-test \
|
||||
"${CACHE_FROM_ARGS[@]}" \
|
||||
|
||||
+65
-4
@@ -100,13 +100,33 @@ ENV UV_HTTP_TIMEOUT=500
|
||||
ENV UV_INDEX_STRATEGY="unsafe-best-match"
|
||||
ENV UV_LINK_MODE="copy"
|
||||
|
||||
# PyTorch provides its own indexes for standard and nightly builds.
|
||||
# PYTORCH_NIGHTLY=1 builds the CPU image against torch nightly (parity with the
|
||||
# GPU torch-nightly lane); default 0 keeps the pinned release CPU build.
|
||||
ARG PYTORCH_NIGHTLY=0
|
||||
ARG PYTORCH_CPU_INDEX_BASE_URL=https://download.pytorch.org/whl
|
||||
|
||||
# Copy requirements files for installation
|
||||
COPY requirements/common.txt requirements/common.txt
|
||||
COPY requirements/cpu.txt requirements/cpu.txt
|
||||
COPY use_existing_torch.py use_existing_torch.py
|
||||
|
||||
# For nightly: install torch nightly (CPU) first, strip torch pins from the
|
||||
# requirements via use_existing_torch.py so they can't pull it back to a release
|
||||
# build, then install the rest from the nightly CPU index (no --torch-backend,
|
||||
# which would force the stable channel). Release path is unchanged.
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv pip install --upgrade pip && \
|
||||
uv pip install -r requirements/cpu.txt --torch-backend cpu
|
||||
if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
|
||||
echo "Installing torch nightly (CPU)..." \
|
||||
&& uv pip install torch torchaudio torchvision torchcodec --pre \
|
||||
--index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu \
|
||||
&& python3 use_existing_torch.py --prefix \
|
||||
&& uv pip install -r requirements/cpu.txt \
|
||||
--extra-index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu; \
|
||||
else \
|
||||
uv pip install -r requirements/cpu.txt --torch-backend cpu; \
|
||||
fi
|
||||
|
||||
ENV TARGETARCH=${TARGETARCH}
|
||||
|
||||
@@ -191,6 +211,10 @@ RUN --mount=type=cache,target=/root/.cargo/registry,sharing=locked \
|
||||
# build it never uses.
|
||||
FROM base-arch AS vllm-src
|
||||
|
||||
# Re-declare torch-nightly args (ARGs do not cross FROM)
|
||||
ARG PYTORCH_NIGHTLY=0
|
||||
ARG PYTORCH_CPU_INDEX_BASE_URL=https://download.pytorch.org/whl
|
||||
|
||||
ARG GIT_REPO_CHECK=0
|
||||
# Support for cross-compilation with x86 ISA including AVX2 and AVX512: docker build --build-arg VLLM_CPU_X86="true" ...
|
||||
ARG VLLM_CPU_X86=0
|
||||
@@ -215,7 +239,13 @@ RUN if [ "$TARGETARCH" = "arm64" ] && [ "$VLLM_CPU_X86" != "0" ]; then \
|
||||
COPY requirements/build/cpu.txt requirements/build/cpu.txt
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv pip install -r requirements/build/cpu.txt --torch-backend cpu
|
||||
if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
|
||||
python3 /workspace/use_existing_torch.py --prefix \
|
||||
&& uv pip install -r requirements/build/cpu.txt \
|
||||
--extra-index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu; \
|
||||
else \
|
||||
uv pip install -r requirements/build/cpu.txt --torch-backend cpu; \
|
||||
fi
|
||||
|
||||
COPY . .
|
||||
|
||||
@@ -226,6 +256,18 @@ COPY --from=rust-build /workspace/vllm/_rust_*.so vllm/
|
||||
|
||||
RUN if [ "$GIT_REPO_CHECK" != 0 ]; then bash tools/check_repo.sh ; fi
|
||||
|
||||
# For torch nightly, strip the torch/vision/audio/codec pins from the source
|
||||
# tree's requirements *after* `COPY . .` (which restored the pinned files) so
|
||||
# the wheel built below does not hard-pin the release torch (e.g.
|
||||
# torch==2.11.0+cpu). setup.py reads requirements/cpu.txt into install_requires;
|
||||
# leaving the pin makes `uv pip install dist/*.whl` unsatisfiable against the
|
||||
# nightly index. The nightly torch installed in the base image satisfies the
|
||||
# now-unpinned dependency. Done at the end of vllm-src so vllm-build and
|
||||
# vllm-dev, which both derive from it, inherit the unpinned tree.
|
||||
RUN if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
|
||||
python3 /workspace/use_existing_torch.py --prefix; \
|
||||
fi
|
||||
|
||||
######################### BUILD IMAGE #########################
|
||||
FROM vllm-src AS vllm-build
|
||||
|
||||
@@ -283,6 +325,10 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
######################### TEST DEPS #########################
|
||||
FROM base-arch AS vllm-test-deps
|
||||
|
||||
# Re-declare torch-nightly args (ARGs do not cross FROM)
|
||||
ARG PYTORCH_NIGHTLY=0
|
||||
ARG PYTORCH_CPU_INDEX_BASE_URL=https://download.pytorch.org/whl
|
||||
|
||||
WORKDIR /vllm-workspace
|
||||
|
||||
# Test requirements are compiled from requirements/test/cuda.in into
|
||||
@@ -297,11 +343,21 @@ RUN case "$(uname -m)" in \
|
||||
esac
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv pip install -r requirements/test/cpu.txt --torch-backend cpu
|
||||
if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
|
||||
python3 /workspace/use_existing_torch.py --prefix \
|
||||
&& uv pip install -r requirements/test/cpu.txt \
|
||||
--extra-index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu; \
|
||||
else \
|
||||
uv pip install -r requirements/test/cpu.txt --torch-backend cpu; \
|
||||
fi
|
||||
|
||||
######################### DEV IMAGE #########################
|
||||
FROM vllm-src AS vllm-dev
|
||||
|
||||
# Re-declare torch-nightly args (ARGs do not cross FROM)
|
||||
ARG PYTORCH_NIGHTLY=0
|
||||
ARG PYTORCH_CPU_INDEX_BASE_URL=https://download.pytorch.org/whl
|
||||
|
||||
WORKDIR /vllm-workspace
|
||||
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||
@@ -323,7 +379,12 @@ COPY --from=vllm-test-deps /vllm-workspace/requirements/test/cpu.txt requirement
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv pip install -r requirements/lint.txt && \
|
||||
uv pip install -r requirements/test/cpu.txt --torch-backend cpu && \
|
||||
if [ "${PYTORCH_NIGHTLY}" = "1" ]; then \
|
||||
uv pip install -r requirements/test/cpu.txt \
|
||||
--extra-index-url ${PYTORCH_CPU_INDEX_BASE_URL}/nightly/cpu; \
|
||||
else \
|
||||
uv pip install -r requirements/test/cpu.txt --torch-backend cpu; \
|
||||
fi && \
|
||||
pre-commit install --hook-type pre-commit --hook-type commit-msg
|
||||
|
||||
# Installed after requirements/test/cpu.txt (which pins triton==3.6.0) so the
|
||||
|
||||
@@ -11,10 +11,12 @@ TORCH_LIB_PREFIXES = (
|
||||
"torch=",
|
||||
"torchvision=",
|
||||
"torchaudio=",
|
||||
"torchcodec=",
|
||||
# pyproject.toml
|
||||
'"torch =',
|
||||
'"torchvision =',
|
||||
'"torchaudio =',
|
||||
'"torchcodec =',
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user