diff --git a/.buildkite/image_build/image_build.sh b/.buildkite/image_build/image_build.sh index 6c8d0b9b36..52f120534b 100755 --- a/.buildkite/image_build/image_build.sh +++ b/.buildkite/image_build/image_build.sh @@ -177,6 +177,18 @@ BRANCH=$4 IMAGE_TAG=$5 IMAGE_TAG_LATEST=${6:-} # only used for main branch, optional +# When TORCH_NIGHTLY=1, build the base CI image against PyTorch nightly so the +# entire existing pipeline runs on nightly torch (CUDA/GPU lane only). Delegate +# to the dedicated nightly build (PYTORCH_NIGHTLY=1, CUDA 13.0) and tag it at the +# normal IMAGE_TAG that every test step already pulls -- no separate image tag, +# no duplicate "vLLM Against PyTorch Nightly" pipeline section. +if [[ "${TORCH_NIGHTLY:-0}" == "1" ]]; then + echo "--- :warning: TORCH_NIGHTLY=1 -- building base image on PyTorch nightly" + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + exec "${SCRIPT_DIR}/image_build_torch_nightly.sh" \ + "${REGISTRY}" "${REPO}" "${BUILDKITE_COMMIT}" "${BRANCH}" "${IMAGE_TAG}" +fi + # build config TARGET="test-ci" VLLM_BAKE_FILE_PATH="${VLLM_BAKE_FILE_PATH:-docker/docker-bake.hcl}" diff --git a/.buildkite/scripts/check-ray-compatibility.sh b/.buildkite/scripts/check-ray-compatibility.sh index b056d4403d..ec42834802 100644 --- a/.buildkite/scripts/check-ray-compatibility.sh +++ b/.buildkite/scripts/check-ray-compatibility.sh @@ -29,7 +29,11 @@ if python3 -c "import torch; assert torch.version.hip" 2>/dev/null; then TORCH_INDEX_URL="" fi else - TORCH_INDEX_URL="https://download.pytorch.org/whl/cu130" + if [ "${TORCH_NIGHTLY:-0}" = "1" ]; then + TORCH_INDEX_URL="https://download.pytorch.org/whl/nightly/cu130" + else + TORCH_INDEX_URL="https://download.pytorch.org/whl/cu130" + fi fi echo ">>> Using PyTorch index: ${TORCH_INDEX_URL:-PyPI default}" diff --git a/.buildkite/scripts/trigger-ci-build.sh b/.buildkite/scripts/trigger-ci-build.sh index 65a761588c..a56683e960 100755 --- a/.buildkite/scripts/trigger-ci-build.sh +++ b/.buildkite/scripts/trigger-ci-build.sh @@ -23,6 +23,7 @@ NC='\033[0m' # No Color # Default configuration PIPELINE="ci" DRY_RUN=true +TORCH_NIGHTLY=false usage() { cat <= ${TORCH_SUPPORTED_VERSION_ROCM} " "expected for ROCm build, saw ${Torch_VERSION} instead.") diff --git a/docs/contributing/ci/update_pytorch_version.md b/docs/contributing/ci/update_pytorch_version.md index 98947dd440..1c29bf2438 100644 --- a/docs/contributing/ci/update_pytorch_version.md +++ b/docs/contributing/ci/update_pytorch_version.md @@ -93,6 +93,28 @@ To address this, manually trigger a build on Buildkite to accomplish two objecti Buildkite new build popup

+You can also trigger this build from the command line with +[`.buildkite/scripts/trigger-ci-build.sh`](../../../.buildkite/scripts/trigger-ci-build.sh) +(dry-run by default; pass `--execute` to actually trigger it). + +## Test against PyTorch nightly + +The steps above test against a specific PyTorch RC/stable wheel pinned in the +requirements files. To instead build and run the CI suite against the latest +PyTorch **nightly** wheels, set the `TORCH_NIGHTLY=1` environment variable on +the build (or apply the `ready-torch-nightly` label to the PR). + +When `TORCH_NIGHTLY=1`, the base CI image is built against PyTorch nightly +(`image_build_torch_nightly.sh`, `PYTORCH_NIGHTLY=1`, CUDA 13.0) and tagged at +the normal image tag, so the entire existing pipeline runs on nightly torch -- +there is no separate pipeline section to trigger. Combine it with `RUN_ALL=1` +to run the full suite (the `ready-torch-nightly` label and +`trigger-ci-build.sh --torch-nightly` both set this for you). This is the +configuration to use for a scheduled "vLLM vs PyTorch nightly" run. + +Use `.buildkite/scripts/trigger-ci-build.sh --torch-nightly` to trigger it from +the command line. + ## Update all the different vLLM platforms Rather than attempting to update all vLLM platforms in a single pull request, it's more manageable