From 867ff69733dd32467a18fed288592373225d8db1 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Fri, 17 Jul 2026 02:16:44 -0700 Subject: [PATCH] [CI] Gate non-default release wheel builds (#48772) Signed-off-by: khluu Co-authored-by: OpenAI Codex --- .buildkite/release-pipeline.yaml | 95 +++++++++++++------ .../generate-and-upload-nightly-index.sh | 8 +- .buildkite/scripts/upload-rocm-wheels.sh | 6 +- docs/contributing/ci/nightly_builds.md | 6 +- 4 files changed, 77 insertions(+), 38 deletions(-) diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml index f65130c319..136c96d218 100644 --- a/.buildkite/release-pipeline.yaml +++ b/.buildkite/release-pipeline.yaml @@ -31,8 +31,46 @@ steps: - text: "What is the release version?" key: release-version - - group: "Build Python wheels" + - group: "Build CUDA 13.0 Python wheels" key: "build-wheels" + steps: + - label: "Build wheel - aarch64 - CUDA 13.0" + depends_on: ~ + id: build-wheel-arm64-cuda-13-0 + agents: + queue: arm64_cpu_queue_release + commands: + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg BUILD_OS=manylinux --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.0 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." + - "mkdir artifacts" + - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" + - "bash .buildkite/scripts/upload-nightly-wheels.sh" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels' + env: + DOCKER_BUILDKIT: "1" + + - label: "Build wheel - x86_64 - CUDA 13.0" + depends_on: ~ + id: build-wheel-x86-cuda-13-0 + agents: + queue: cpu_queue_release + commands: + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg BUILD_OS=manylinux --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.0 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." + - "mkdir artifacts" + - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" + - "bash .buildkite/scripts/upload-nightly-wheels.sh" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels' + env: + DOCKER_BUILDKIT: "1" + + - block: "Unblock to build additional Python wheels" + depends_on: ~ + key: block-build-additional-wheels + if: build.env("NIGHTLY") != "1" + + - group: "Build additional Python wheels" + key: "build-additional-wheels" + depends_on: block-build-additional-wheels + allow_dependency_failure: true steps: - label: "Build wheel - aarch64 - CUDA 12.9" depends_on: ~ @@ -48,20 +86,6 @@ steps: env: DOCKER_BUILDKIT: "1" - - label: "Build wheel - aarch64 - CUDA 13.0" - depends_on: ~ - id: build-wheel-arm64-cuda-13-0 - agents: - queue: arm64_cpu_queue_release - commands: - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg BUILD_OS=manylinux --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.0 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." - - "mkdir artifacts" - - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - - "bash .buildkite/scripts/upload-nightly-wheels.sh" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels' - env: - DOCKER_BUILDKIT: "1" - - label: "Build wheel - aarch64 - CPU" depends_on: ~ id: build-wheel-arm64-cpu @@ -133,20 +157,6 @@ steps: env: DOCKER_BUILDKIT: "1" - - label: "Build wheel - x86_64 - CUDA 13.0" - depends_on: ~ - id: build-wheel-x86-cuda-13-0 - agents: - queue: cpu_queue_release - commands: - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg BUILD_OS=manylinux --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.0 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ." - - "mkdir artifacts" - - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'" - - "bash .buildkite/scripts/upload-nightly-wheels.sh" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels' - env: - DOCKER_BUILDKIT: "1" - - label: "Build wheel - x86_64 - CPU" depends_on: ~ id: build-wheel-x86-cpu @@ -162,12 +172,26 @@ steps: DOCKER_BUILDKIT: "1" - label: "Generate and upload wheel indices" + key: generate-wheel-indices depends_on: "build-wheels" allow_dependency_failure: true + if: build.env("NIGHTLY") != "1" + agents: + queue: cpu_queue_release + commands: + - "UPDATE_VERSION_INDEX=0 bash .buildkite/scripts/generate-and-upload-nightly-index.sh" + + - label: "Regenerate indices with additional wheels" + key: generate-additional-wheel-indices + depends_on: + - build-wheels + - build-additional-wheels + - generate-wheel-indices + allow_dependency_failure: true agents: queue: cpu_queue_release commands: - - "bash .buildkite/scripts/generate-and-upload-nightly-index.sh" + - 'UPDATE_NIGHTLY_INDEX="$${NIGHTLY:-0}" bash .buildkite/scripts/generate-and-upload-nightly-index.sh' - block: "Unblock to build release Docker images" depends_on: ~ @@ -566,10 +590,17 @@ steps: # # ============================================================================= + - block: "Unblock ROCm wheel/image prerequisites" + depends_on: ~ + key: block-build-rocm + if: build.env("NIGHTLY") != "1" + # ROCm Job 1: Build ROCm Base Wheels (with S3 caching) - label: ":rocm: Build ROCm Base Image & Wheels" id: build-rocm-base-wheels - depends_on: ~ + depends_on: + - step: block-build-rocm + allow_failure: true agents: queue: cpu_queue_release commands: @@ -974,6 +1005,8 @@ steps: depends_on: - input-release-version - build-wheels + - build-additional-wheels + - generate-additional-wheel-indices - label: "Upload release wheels to PyPI" depends_on: diff --git a/.buildkite/scripts/generate-and-upload-nightly-index.sh b/.buildkite/scripts/generate-and-upload-nightly-index.sh index 502ed06093..1fa75994c0 100755 --- a/.buildkite/scripts/generate-and-upload-nightly-index.sh +++ b/.buildkite/scripts/generate-and-upload-nightly-index.sh @@ -45,8 +45,10 @@ $PYTHON .buildkite/scripts/generate-nightly-index.py --version "$SUBPATH" --curr echo "Uploading indices to $S3_COMMIT_PREFIX" aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "$S3_COMMIT_PREFIX" -# copy to /nightly/ only if it is on the main branch and not a PR -if [[ "$BUILDKITE_BRANCH" == "main" && "$BUILDKITE_PULL_REQUEST" == "false" ]]; then +# copy to /nightly/ only when enabled for a main branch build that is not a PR +if [[ "${UPDATE_NIGHTLY_INDEX:-1}" == "1" && \ + "$BUILDKITE_BRANCH" == "main" && \ + "$BUILDKITE_PULL_REQUEST" == "false" ]]; then echo "Uploading indices to overwrite /nightly/" aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/nightly/" fi @@ -67,7 +69,7 @@ pure_version="${version%%+*}" echo "Pure version (without variant): $pure_version" # re-generate and copy to // only if it does not have "dev" in the version -if [[ "$version" != *"dev"* ]]; then +if [[ "${UPDATE_VERSION_INDEX:-1}" == "1" && "$version" != *"dev"* ]]; then echo "Re-generating indices for /$pure_version/" rm -rf "${INDICES_OUTPUT_DIR:?}" mkdir -p "$INDICES_OUTPUT_DIR" diff --git a/.buildkite/scripts/upload-rocm-wheels.sh b/.buildkite/scripts/upload-rocm-wheels.sh index 1f36556312..65fca944c7 100755 --- a/.buildkite/scripts/upload-rocm-wheels.sh +++ b/.buildkite/scripts/upload-rocm-wheels.sh @@ -113,8 +113,8 @@ $PYTHON .buildkite/scripts/generate-nightly-index.py \ echo "Uploading indices to $S3_COMMIT_PREFIX" aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "$S3_COMMIT_PREFIX" -# Update rocm/nightly/ if on main branch and not a PR -if [[ "$BUILDKITE_BRANCH" == "main" && "$BUILDKITE_PULL_REQUEST" == "false" ]] || [[ "$NIGHTLY" == "1" ]]; then +# Only scheduled nightly builds should update the moving nightly index. +if [[ "${NIGHTLY:-0}" == "1" ]]; then echo "Updating rocm/nightly/ index..." aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/rocm/nightly/" fi @@ -147,7 +147,7 @@ echo "" echo "Install command (by commit):" echo " pip install vllm --extra-index-url https://${BUCKET}.s3.amazonaws.com/$ROCM_SUBPATH/" echo "" -if [[ "$BUILDKITE_BRANCH" == "main" ]] || [[ "$NIGHTLY" == "1" ]]; then +if [[ "${NIGHTLY:-0}" == "1" ]]; then echo "Install command (nightly):" echo " pip install vllm --extra-index-url https://${BUCKET}.s3.amazonaws.com/rocm/nightly/" fi diff --git a/docs/contributing/ci/nightly_builds.md b/docs/contributing/ci/nightly_builds.md index 10c4a43724..c6d83fd9fc 100644 --- a/docs/contributing/ci/nightly_builds.md +++ b/docs/contributing/ci/nightly_builds.md @@ -6,7 +6,11 @@ vLLM maintains a per-commit wheel repository (commonly referred to as "nightly") ### Wheel Building -Wheels are built in the `Release` pipeline (`.buildkite/release-pipeline.yaml`) after a PR is merged into the main branch, with multiple variants: +Wheels are built in the `Release` pipeline +(`.buildkite/release-pipeline.yaml`) after a PR is merged into the main branch. +Regular builds produce the CUDA 13.0 wheels for x86_64 and aarch64. Additional +wheel variants and ROCm builds can be unblocked on demand and run automatically +when `NIGHTLY=1`: - **Backend variants**: `cpu` and `cuXXX` (e.g., `cu129`, `cu130`). - **Architecture variants**: `x86_64` and `aarch64`.