[CI] Add TORCH_NIGHTLY=1 build mode (run full suite on torch nightly) (#47180)
Co-authored-by: Andrey Talman <atalman@users.noreply.github.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Andrey Talman
Kevin H. Luu
parent
424df4f65d
commit
f36284a8d2
@@ -177,6 +177,18 @@ BRANCH=$4
|
||||
IMAGE_TAG=$5
|
||||
IMAGE_TAG_LATEST=${6:-} # only used for main branch, optional
|
||||
|
||||
# When TORCH_NIGHTLY=1, build the base CI image against PyTorch nightly so the
|
||||
# entire existing pipeline runs on nightly torch (CUDA/GPU lane only). Delegate
|
||||
# to the dedicated nightly build (PYTORCH_NIGHTLY=1, CUDA 13.0) and tag it at the
|
||||
# normal IMAGE_TAG that every test step already pulls -- no separate image tag,
|
||||
# no duplicate "vLLM Against PyTorch Nightly" pipeline section.
|
||||
if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then
|
||||
echo "--- :warning: TORCH_NIGHTLY=1 -- building base image on PyTorch nightly"
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
exec "${SCRIPT_DIR}/image_build_torch_nightly.sh" \
|
||||
"${REGISTRY}" "${REPO}" "${BUILDKITE_COMMIT}" "${BRANCH}" "${IMAGE_TAG}"
|
||||
fi
|
||||
|
||||
# build config
|
||||
TARGET="test-ci"
|
||||
VLLM_BAKE_FILE_PATH="${VLLM_BAKE_FILE_PATH:-docker/docker-bake.hcl}"
|
||||
|
||||
@@ -29,7 +29,11 @@ if python3 -c "import torch; assert torch.version.hip" 2>/dev/null; then
|
||||
TORCH_INDEX_URL=""
|
||||
fi
|
||||
else
|
||||
TORCH_INDEX_URL="https://download.pytorch.org/whl/cu130"
|
||||
if [ "${TORCH_NIGHTLY:-0}" = "1" ]; then
|
||||
TORCH_INDEX_URL="https://download.pytorch.org/whl/nightly/cu130"
|
||||
else
|
||||
TORCH_INDEX_URL="https://download.pytorch.org/whl/cu130"
|
||||
fi
|
||||
fi
|
||||
echo ">>> Using PyTorch index: ${TORCH_INDEX_URL:-PyPI default}"
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ NC='\033[0m' # No Color
|
||||
# Default configuration
|
||||
PIPELINE="ci"
|
||||
DRY_RUN=true
|
||||
TORCH_NIGHTLY=false
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
@@ -34,12 +35,14 @@ Sets RUN_ALL=1 and NIGHTLY=1 environment variables.
|
||||
SAFETY: Dry-run by default. Use --execute to actually trigger a build.
|
||||
|
||||
Options:
|
||||
--execute Actually trigger the build (default: dry-run)
|
||||
--pipeline Buildkite pipeline slug (default: ${PIPELINE})
|
||||
--commit Override commit SHA (default: current HEAD)
|
||||
--branch Override branch name (default: current branch)
|
||||
--message Custom build message (default: auto-generated)
|
||||
--help Show this help message
|
||||
--execute Actually trigger the build (default: dry-run)
|
||||
--pipeline Buildkite pipeline slug (default: ${PIPELINE})
|
||||
--commit Override commit SHA (default: current HEAD)
|
||||
--branch Override branch name (default: current branch)
|
||||
--message Custom build message (default: auto-generated)
|
||||
--torch-nightly Also build and run the full suite against torch nightly
|
||||
(sets TORCH_NIGHTLY=1)
|
||||
--help Show this help message
|
||||
|
||||
Prerequisites:
|
||||
- bk CLI installed: brew tap buildkite/buildkite && brew install buildkite/buildkite/bk
|
||||
@@ -49,6 +52,7 @@ Examples:
|
||||
$(basename "$0") # Dry-run, show what would happen
|
||||
$(basename "$0") --execute # Actually trigger the build
|
||||
$(basename "$0") --pipeline ci-shadow # Dry-run with different pipeline
|
||||
$(basename "$0") --torch-nightly # Dry-run a full torch-nightly run
|
||||
EOF
|
||||
exit 1
|
||||
}
|
||||
@@ -96,6 +100,10 @@ while [[ $# -gt 0 ]]; do
|
||||
MESSAGE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--torch-nightly)
|
||||
TORCH_NIGHTLY=true
|
||||
shift
|
||||
;;
|
||||
--help|-h)
|
||||
usage
|
||||
;;
|
||||
@@ -171,11 +179,17 @@ if [[ $(echo "$REMOTE_BRANCHES" | wc -l) -gt 5 ]]; then
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Environment variables passed to the build.
|
||||
BUILD_ENV=("RUN_ALL=1" "NIGHTLY=1")
|
||||
if [[ "$TORCH_NIGHTLY" == true ]]; then
|
||||
BUILD_ENV+=("TORCH_NIGHTLY=1")
|
||||
fi
|
||||
|
||||
log_info "Pipeline: ${PIPELINE}"
|
||||
log_info "Branch: ${BRANCH}"
|
||||
log_info "Commit: ${COMMIT}"
|
||||
log_info "Message: ${MESSAGE}"
|
||||
log_info "Environment: RUN_ALL=1, NIGHTLY=1"
|
||||
log_info "Environment: ${BUILD_ENV[*]}"
|
||||
echo ""
|
||||
|
||||
# Build the command
|
||||
@@ -187,9 +201,10 @@ CMD=(bk build create
|
||||
--commit "${COMMIT}"
|
||||
--branch "${BRANCH}"
|
||||
--message "${MESSAGE}"
|
||||
--env "RUN_ALL=1"
|
||||
--env "NIGHTLY=1"
|
||||
)
|
||||
for env_var in "${BUILD_ENV[@]}"; do
|
||||
CMD+=(--env "${env_var}")
|
||||
done
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "=========================================="
|
||||
@@ -210,8 +225,14 @@ if [[ "$DRY_RUN" == true ]]; then
|
||||
echo " --commit '$(escape_for_shell "${COMMIT}")' \\"
|
||||
echo " --branch '$(escape_for_shell "${BRANCH}")' \\"
|
||||
echo " --message '$(escape_for_shell "${MESSAGE}")' \\"
|
||||
echo " --env 'RUN_ALL=1' \\"
|
||||
echo " --env 'NIGHTLY=1'"
|
||||
last_idx=$(( ${#BUILD_ENV[@]} - 1 ))
|
||||
for i in "${!BUILD_ENV[@]}"; do
|
||||
if [[ $i -eq $last_idx ]]; then
|
||||
echo " --env '$(escape_for_shell "${BUILD_ENV[$i]}")'"
|
||||
else
|
||||
echo " --env '$(escape_for_shell "${BUILD_ENV[$i]}")' \\"
|
||||
fi
|
||||
done
|
||||
echo ""
|
||||
echo "=========================================="
|
||||
echo -e "${YELLOW}To actually trigger this build, run:${NC}"
|
||||
|
||||
+11
-2
@@ -70,6 +70,15 @@ endif()
|
||||
#
|
||||
set(TORCH_SUPPORTED_VERSION_CUDA "2.11.0")
|
||||
set(TORCH_SUPPORTED_VERSION_ROCM "2.11.0")
|
||||
# TORCH_NIGHTLY=1 builds run against unpinned nightly wheels, so the supported-
|
||||
# version check would always warn. Only treat it as a nightly build when the
|
||||
# value is exactly "1" (the bootstrap exports TORCH_NIGHTLY=0 by default, which
|
||||
# must NOT suppress the warning for normal builds).
|
||||
if (DEFINED ENV{TORCH_NIGHTLY} AND "$ENV{TORCH_NIGHTLY}" STREQUAL "1")
|
||||
set(TORCH_NIGHTLY_BUILD TRUE)
|
||||
else()
|
||||
set(TORCH_NIGHTLY_BUILD FALSE)
|
||||
endif()
|
||||
|
||||
#
|
||||
# Try to find python package with an executable that exactly matches
|
||||
@@ -175,7 +184,7 @@ endif()
|
||||
if (NOT HIP_FOUND AND NOT PYTORCH_FOUND_HIP AND CUDA_FOUND)
|
||||
set(VLLM_GPU_LANG "CUDA")
|
||||
|
||||
if (NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_CUDA})
|
||||
if (NOT TORCH_NIGHTLY_BUILD AND NOT Torch_VERSION VERSION_EQUAL ${TORCH_SUPPORTED_VERSION_CUDA})
|
||||
message(WARNING "Pytorch version ${TORCH_SUPPORTED_VERSION_CUDA} "
|
||||
"expected for CUDA build, saw ${Torch_VERSION} instead.")
|
||||
endif()
|
||||
@@ -188,7 +197,7 @@ elseif(HIP_FOUND OR PYTORCH_FOUND_HIP)
|
||||
enable_language(HIP)
|
||||
|
||||
# ROCm 5.X and 6.X
|
||||
if (ROCM_VERSION_DEV_MAJOR GREATER_EQUAL 5 AND
|
||||
if (NOT TORCH_NIGHTLY_BUILD AND ROCM_VERSION_DEV_MAJOR GREATER_EQUAL 5 AND
|
||||
Torch_VERSION VERSION_LESS ${TORCH_SUPPORTED_VERSION_ROCM})
|
||||
message(WARNING "Pytorch version >= ${TORCH_SUPPORTED_VERSION_ROCM} "
|
||||
"expected for ROCm build, saw ${Torch_VERSION} instead.")
|
||||
|
||||
@@ -93,6 +93,28 @@ To address this, manually trigger a build on Buildkite to accomplish two objecti
|
||||
<img width="60%" alt="Buildkite new build popup" src="https://github.com/user-attachments/assets/3b07f71b-bb18-4ca3-aeaf-da0fe79d315f" />
|
||||
</p>
|
||||
|
||||
You can also trigger this build from the command line with
|
||||
[`.buildkite/scripts/trigger-ci-build.sh`](../../../.buildkite/scripts/trigger-ci-build.sh)
|
||||
(dry-run by default; pass `--execute` to actually trigger it).
|
||||
|
||||
## Test against PyTorch nightly
|
||||
|
||||
The steps above test against a specific PyTorch RC/stable wheel pinned in the
|
||||
requirements files. To instead build and run the CI suite against the latest
|
||||
PyTorch **nightly** wheels, set the `TORCH_NIGHTLY=1` environment variable on
|
||||
the build (or apply the `ready-torch-nightly` label to the PR).
|
||||
|
||||
When `TORCH_NIGHTLY=1`, the base CI image is built against PyTorch nightly
|
||||
(`image_build_torch_nightly.sh`, `PYTORCH_NIGHTLY=1`, CUDA 13.0) and tagged at
|
||||
the normal image tag, so the entire existing pipeline runs on nightly torch --
|
||||
there is no separate pipeline section to trigger. Combine it with `RUN_ALL=1`
|
||||
to run the full suite (the `ready-torch-nightly` label and
|
||||
`trigger-ci-build.sh --torch-nightly` both set this for you). This is the
|
||||
configuration to use for a scheduled "vLLM vs PyTorch nightly" run.
|
||||
|
||||
Use `.buildkite/scripts/trigger-ci-build.sh --torch-nightly` to trigger it from
|
||||
the command line.
|
||||
|
||||
## Update all the different vLLM platforms
|
||||
|
||||
Rather than attempting to update all vLLM platforms in a single pull request, it's more manageable
|
||||
|
||||
Reference in New Issue
Block a user