vLLM/.buildkite/release-pipeline.yaml

# CUDA architecture lists — following PyTorch RELEASE.md
# (https://github.com/pytorch/pytorch/blob/main/RELEASE.md)
# SM86 included for broader Ampere coverage; SM89 for marlin fp8 support
# These requested arches are filtered by CMake's CUDA_SUPPORTED_ARCHS before
# per-kernel arch selection. Do not add +PTX here: top-level +PTX is stripped
# during that filtering, so kernels that need PTX must request it locally.
env:
  # for CUDA >=13, sm_100+ targets have family specifiers (see CMakeLists.txt)
  # so targets like 10.3 and 12.1 are automatically supported with this list
  CUDA_ARCH_X86: "7.5 8.0 8.6 8.9 9.0 10.0 12.0"
  # aarch64-only targets: Orin (8.7), Thor (11.0, CUDA 13+)
  CUDA_ARCH_AARCH64: "8.0 8.7 8.9 9.0 10.0 11.0 12.0"

  # for CUDA <13, we need to specify all needed targets
  # some targets (10.3, 12.1) are skipped to limit the wheel size (< 500MB)
  # please use CUDA 13 wheels or compile yourself on these new devices
  CUDA_ARCH_X86_CU129: "7.5 8.0 8.6 8.9 9.0 10.0 12.0"
  CUDA_ARCH_AARCH64_CU129: "8.0 8.7 8.9 9.0 10.0 12.0"

steps:
  - input: "Provide Release version here"
    id: input-release-version
    fields:
      - text: "What is the release version?"
        key: release-version

  - group: "Build CUDA 13.0 Python wheels"
    key: "build-wheels"
    steps:
      - label: "Build wheel - aarch64 - CUDA 13.0"
        depends_on: ~
        id: build-wheel-arm64-cuda-13-0
        agents:
          queue: arm64_cpu_queue_release
        commands:
          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64}\" --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.0-b8b5f17a7d9ccfc25bbc5cf17b3fcea12964a042 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
          - "mkdir artifacts"
          - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
          - "bash .buildkite/scripts/upload-nightly-wheels.sh"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels'
        env:
          DOCKER_BUILDKIT: "1"

      - label: "Build wheel - x86_64 - CUDA 13.0"
        depends_on: ~
        id: build-wheel-x86-cuda-13-0
        agents:
          queue: cpu_queue_release
        commands:
          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=13.0.2 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86}\" --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.0-78e737ad29420ffc4800e677c51e2a852caf8359 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
          - "mkdir artifacts"
          - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
          - "bash .buildkite/scripts/upload-nightly-wheels.sh"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels'
        env:
          DOCKER_BUILDKIT: "1"

  - block: "Unblock to build additional Python wheels"
    depends_on: ~
    key: block-build-additional-wheels
    if: build.env("NIGHTLY") != "1"

  - group: "Build additional Python wheels"
    key: "build-additional-wheels"
    depends_on: block-build-additional-wheels
    allow_dependency_failure: true
    steps:
      - label: "Build wheel - aarch64 - CUDA 12.9"
        depends_on: ~
        id: build-wheel-arm64-cuda-12-9
        agents:
          queue: arm64_cpu_queue_release
        commands:
          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_AARCH64_CU129}\" --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda12.9-78e737ad29420ffc4800e677c51e2a852caf8359 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
          - "mkdir artifacts"
          - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
          - "bash .buildkite/scripts/upload-nightly-wheels.sh"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels'
        env:
          DOCKER_BUILDKIT: "1"

      - label: "Build wheel - aarch64 - CPU"
        depends_on: ~
        id: build-wheel-arm64-cpu
        agents:
          queue: arm64_cpu_queue_release
        commands:
          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_BUILD_ACL=ON --tag vllm-ci:build-image --target vllm-build --progress plain -f docker/Dockerfile.cpu ."
          - "mkdir artifacts"
          - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
          - "bash .buildkite/scripts/upload-nightly-wheels.sh"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels'
        env:
          DOCKER_BUILDKIT: "1"

      - label: "Build wheel - macOS arm64 - CPU"
        depends_on: ~
        id: build-wheel-macos-arm64-cpu
        agents:
          queue: macmini
        timeout_in_minutes: 90
        commands:
          - "bash .buildkite/scripts/build-macos-wheel.sh"
          - "mkdir -p artifacts/transfer"
          - "basename artifacts/dist/*.whl > artifacts/transfer/wheel.name"
          - "shasum -a 256 artifacts/dist/*.whl > artifacts/transfer/wheel.sha256"
          - "split -b 1m artifacts/dist/*.whl artifacts/transfer/wheel.part."
        env:
          BUILDKITE_ARTIFACT_UPLOAD_CONCURRENCY: "1"
          BUILDKITE_NO_HTTP2: "true"
        artifact_paths:
          - "artifacts/transfer/*"

      - label: "Upload wheel - macOS arm64 - CPU"
        depends_on: build-wheel-macos-arm64-cpu
        id: upload-wheel-macos-arm64-cpu
        agents:
          queue: small_cpu_queue_release
        timeout_in_minutes: 15
        commands:
          - 'buildkite-agent artifact download "artifacts/transfer/*" .'
          - "mkdir -p artifacts/dist artifacts/reassembled"
          - 'wheel_name=$$(cat artifacts/transfer/wheel.name)'
          - '[[ "$$wheel_name" =~ ^[A-Za-z0-9][A-Za-z0-9._+-]*\.whl$$ && "$$wheel_name" != *..* ]]'
          - 'expected_sha=$$(awk ''{print $$1}'' artifacts/transfer/wheel.sha256)'
          - '[[ "$$expected_sha" =~ ^[0-9a-f]{64}$$ ]]'
          - 'cat artifacts/transfer/wheel.part.* > artifacts/reassembled/wheel'
          - 'actual_sha=$$(shasum -a 256 artifacts/reassembled/wheel | awk ''{print $$1}'')'
          - '[[ "$$actual_sha" == "$$expected_sha" ]]'
          - 'mv artifacts/reassembled/wheel "artifacts/dist/$$wheel_name"'
          - "aws sts get-caller-identity"
          - "VLLM_WHEEL_PLATFORM=macos bash .buildkite/scripts/upload-nightly-wheels.sh"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels'
        plugins:
          - aws-assume-role-with-web-identity#v1.6.0:
              role-arn: arn:aws:iam::936637512419:role/vllm-release-macos-wheel-uploader
              region: us-west-2

      - label: "Build wheel - x86_64 - CUDA 12.9"
        depends_on: ~
        id: build-wheel-x86-cuda-12-9
        agents:
          queue: cpu_queue_release
        commands:
          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg USE_SCCACHE=1 --build-arg GIT_REPO_CHECK=1 --build-arg CUDA_VERSION=12.9.1 --build-arg torch_cuda_arch_list=\"${CUDA_ARCH_X86_CU129}\" --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda12.9-78e737ad29420ffc4800e677c51e2a852caf8359 --tag vllm-ci:build-image --target build --progress plain -f docker/Dockerfile ."
          - "mkdir artifacts"
          - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
          - "bash .buildkite/scripts/upload-nightly-wheels.sh"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels'
        env:
          DOCKER_BUILDKIT: "1"

      - label: "Build wheel - x86_64 - CPU"
        depends_on: ~
        id: build-wheel-x86-cpu
        agents:
          queue: cpu_queue_release
        commands:
          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_CPU_X86=true --tag vllm-ci:build-image --target vllm-build --progress plain -f docker/Dockerfile.cpu ."
          - "mkdir artifacts"
          - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
          - "bash .buildkite/scripts/upload-nightly-wheels.sh"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels'
        env:
          DOCKER_BUILDKIT: "1"

      - label: "Build wheel - x86_64 - XPU"
        depends_on: ~
        id: build-wheel-x86-xpu
        agents:
          queue: cpu_queue_release
        commands:
          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --tag vllm-ci:build-image --target vllm-build --progress plain -f docker/Dockerfile.xpu ."
          - "mkdir artifacts"
          - "docker run --rm -v $(pwd)/artifacts:/artifacts_host vllm-ci:build-image bash -c 'cp -r dist /artifacts_host && chmod -R a+rw /artifacts_host'"
          - "bash .buildkite/scripts/upload-nightly-wheels.sh"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "s3://vllm-wheels/$$BUILDKITE_COMMIT/$(cd artifacts/dist && echo *.whl)" release-wheels'
        env:
          DOCKER_BUILDKIT: "1"

  - label: "Publish and stage XPU Triton shim"
    key: publish-xpu-triton-shim
    depends_on: ~
    agents:
      queue: cpu_queue_release
    commands:
      - "bash .buildkite/scripts/xpu/publish-triton-shim.sh"

  - label: "Generate and upload wheel indices"
    key: generate-wheel-indices
    depends_on: "build-wheels"
    allow_dependency_failure: true
    if: build.env("NIGHTLY") != "1"
    agents:
      queue: cpu_queue_release
    commands:
      - "UPDATE_VERSION_INDEX=0 bash .buildkite/scripts/generate-and-upload-nightly-index.sh"

  - label: "Regenerate indices with additional wheels"
    key: generate-additional-wheel-indices
    depends_on:
      - build-wheels
      - build-additional-wheels
      - generate-wheel-indices
    allow_dependency_failure: true
    agents:
      queue: cpu_queue_release
    commands:
      - 'UPDATE_NIGHTLY_INDEX="$${NIGHTLY:-0}" bash .buildkite/scripts/generate-and-upload-nightly-index.sh'

  - block: "Unblock to build release Docker images"
    depends_on: ~
    key: block-build-release-images
    if: build.env("NIGHTLY") != "1"

  - group: "Build release Docker images"
    key: "build-release-images"
    depends_on: block-build-release-images
    allow_dependency_failure: true
    steps:
      - label: "Build release image - x86_64 - CUDA 13.0"
        depends_on: ~
        id: build-release-image-x86
        agents:
          queue: cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - |
            DOCKER_BUILDKIT=1 docker build \
              $(bash .buildkite/scripts/docker-build-metadata-args.sh) \
              --build-arg max_jobs=16 \
              --build-arg USE_SCCACHE=1 \
              --build-arg GIT_REPO_CHECK=1 \
              --build-arg CUDA_VERSION=13.0.2 \
              --build-arg torch_cuda_arch_list="${CUDA_ARCH_X86}" \
              --build-arg INSTALL_KV_CONNECTORS=true \
              --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.0-78e737ad29420ffc4800e677c51e2a852caf8359 \
              --target vllm-openai \
              --progress plain \
              -f docker/Dockerfile .
          - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
          # re-tag to default image tag and push, just in case arm64 build fails
          - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m) public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT"
          - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)" release-images'

      - label: "Build release image - aarch64 - CUDA 13.0"
        depends_on: ~
        id: build-release-image-arm64
        agents:
          queue: arm64_cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - |
            DOCKER_BUILDKIT=1 docker build \
              $(bash .buildkite/scripts/docker-build-metadata-args.sh) \
              --build-arg max_jobs=16 \
              --build-arg USE_SCCACHE=1 \
              --build-arg GIT_REPO_CHECK=1 \
              --build-arg CUDA_VERSION=13.0.2 \
              --build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64}" \
              --build-arg INSTALL_KV_CONNECTORS=true \
              --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.0-b8b5f17a7d9ccfc25bbc5cf17b3fcea12964a042 \
              --target vllm-openai \
              --progress plain \
              -f docker/Dockerfile .
          - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)" release-images'

      - label: "Build release image - x86_64 - CUDA 12.9"
        depends_on: ~
        id: build-release-image-x86-cuda-12-9
        agents:
          queue: cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - |
            DOCKER_BUILDKIT=1 docker build \
              $(bash .buildkite/scripts/docker-build-metadata-args.sh cu129) \
              --build-arg max_jobs=16 \
              --build-arg USE_SCCACHE=1 \
              --build-arg GIT_REPO_CHECK=1 \
              --build-arg CUDA_VERSION=12.9.1 \
              --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda12.9-78e737ad29420ffc4800e677c51e2a852caf8359 \
              --build-arg torch_cuda_arch_list="${CUDA_ARCH_X86_CU129}" \
              --build-arg INSTALL_KV_CONNECTORS=true \
              --target vllm-openai \
              --progress plain \
              -f docker/Dockerfile .
          - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129"
          # re-tag to default image tag and push, just in case arm64 build fails
          - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129"
          - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129" release-images'

      - label: "Build release image - aarch64 - CUDA 12.9"
        depends_on: ~
        id: build-release-image-arm64-cuda-12-9
        agents:
          queue: arm64_cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - |
            DOCKER_BUILDKIT=1 docker build \
              $(bash .buildkite/scripts/docker-build-metadata-args.sh cu129) \
              --build-arg max_jobs=16 \
              --build-arg USE_SCCACHE=1 \
              --build-arg GIT_REPO_CHECK=1 \
              --build-arg CUDA_VERSION=12.9.1 \
              --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda12.9-78e737ad29420ffc4800e677c51e2a852caf8359 \
              --build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64_CU129}" \
              --build-arg INSTALL_KV_CONNECTORS=true \
              --target vllm-openai \
              --progress plain \
              -f docker/Dockerfile .
          - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129" release-images'

      - label: "Build release image - x86_64 - Rubin CUDA 13.4"
        depends_on: ~
        id: build-release-image-x86-rubin-cuda-13-4
        if: build.env("NIGHTLY") == "1"
        agents:
          queue: cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - |
            DOCKER_BUILDKIT=1 docker buildx build --load \
              $(bash .buildkite/scripts/docker-build-metadata-args.sh cu134) \
              --platform linux/amd64 \
              --build-arg max_jobs=16 \
              --build-arg USE_SCCACHE=1 \
              --build-arg GIT_REPO_CHECK=1 \
              --build-arg CUDA_VERSION=13.4 \
              --build-arg torch_cuda_arch_list="${CUDA_ARCH_X86}" \
              --build-arg INSTALL_KV_CONNECTORS=true \
              --build-arg INSTALL_RUBIN_PRERELEASE=true \
              --build-arg TRITON_INSTALL_FROM_SOURCE_REPO=https://github.com/triton-lang/triton.git \
              --build-arg TRITON_INSTALL_FROM_SOURCE_REVISION=3f6e41132b5edf639bfb872ad73d4688765e08b8 \
              --build-arg RUN_WHEEL_CHECK=false \
              --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.4 \
              --build-arg FINAL_BASE_IMAGE=nvidia/cuda:13.4.1-base-ubuntu24.04 \
              --build-arg NCCL_VERSION=2.32.3 \
              --target vllm-openai \
              --progress plain \
              -f docker/Dockerfile .
          - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134" release-images'

      - label: "Build release image - aarch64 - Rubin CUDA 13.4"
        depends_on: ~
        id: build-release-image-arm64-rubin-cuda-13-4
        if: build.env("NIGHTLY") == "1"
        agents:
          queue: arm64_cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          # TODO(arpera):
          # Remove BUILD_NIXL=true once NIXL has a wheel built against torch 2.15
          # For details: https://github.com/vllm-project/vllm/pull/60090
          # Tracking issue report: https://github.com/ai-dynamo/nixl/issues/2297
          - |
            DOCKER_BUILDKIT=1 docker buildx build --load \
              $(bash .buildkite/scripts/docker-build-metadata-args.sh cu134) \
              --platform linux/arm64 \
              --build-arg max_jobs=16 \
              --build-arg USE_SCCACHE=1 \
              --build-arg GIT_REPO_CHECK=1 \
              --build-arg CUDA_VERSION=13.4 \
              --build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64}" \
              --build-arg INSTALL_KV_CONNECTORS=true \
              --build-arg INSTALL_RUBIN_PRERELEASE=true \
              --build-arg BUILD_NIXL=true \
              --build-arg TRITON_INSTALL_FROM_SOURCE_REPO=https://github.com/triton-lang/triton.git \
              --build-arg TRITON_INSTALL_FROM_SOURCE_REVISION=3f6e41132b5edf639bfb872ad73d4688765e08b8 \
              --build-arg RUN_WHEEL_CHECK=false \
              --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.4 \
              --build-arg FINAL_BASE_IMAGE=nvidia/cuda:13.4.1-base-ubuntu24.04 \
              --build-arg NCCL_VERSION=2.32.3 \
              --target vllm-openai \
              --progress plain \
              -f docker/Dockerfile .
          - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu134" release-images'

      - label: "Build release image - x86_64 - CUDA 13.0 - Ubuntu 24.04"
        depends_on: ~
        id: build-release-image-x86-ubuntu2404
        agents:
          queue: cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - |
            DOCKER_BUILDKIT=1 docker build \
              $(bash .buildkite/scripts/docker-build-metadata-args.sh ubuntu2404) \
              --build-arg max_jobs=16 \
              --build-arg USE_SCCACHE=1 \
              --build-arg GIT_REPO_CHECK=1 \
              --build-arg CUDA_VERSION=13.0.2 \
              --build-arg UBUNTU_VERSION=24.04 \
              --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \
              --build-arg torch_cuda_arch_list="${CUDA_ARCH_X86}" \
              --build-arg INSTALL_KV_CONNECTORS=true \
              --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda13.0-78e737ad29420ffc4800e677c51e2a852caf8359 \
              --target vllm-openai \
              --progress plain \
              -f docker/Dockerfile .
          - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404"
          - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404"
          - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404" release-images'

      - label: "Build release image - aarch64 - CUDA 13.0 - Ubuntu 24.04"
        depends_on: ~
        id: build-release-image-arm64-ubuntu2404
        agents:
          queue: arm64_cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - |
            DOCKER_BUILDKIT=1 docker build \
              $(bash .buildkite/scripts/docker-build-metadata-args.sh ubuntu2404) \
              --build-arg max_jobs=16 \
              --build-arg USE_SCCACHE=1 \
              --build-arg GIT_REPO_CHECK=1 \
              --build-arg CUDA_VERSION=13.0.2 \
              --build-arg UBUNTU_VERSION=24.04 \
              --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \
              --build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64}" \
              --build-arg INSTALL_KV_CONNECTORS=true \
              --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda13.0-b8b5f17a7d9ccfc25bbc5cf17b3fcea12964a042 \
              --target vllm-openai \
              --progress plain \
              -f docker/Dockerfile .
          - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-ubuntu2404" release-images'

      - label: "Build release image - x86_64 - CUDA 12.9 - Ubuntu 24.04"
        depends_on: ~
        id: build-release-image-x86-cuda-12-9-ubuntu2404
        agents:
          queue: cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - |
            DOCKER_BUILDKIT=1 docker build \
              $(bash .buildkite/scripts/docker-build-metadata-args.sh cu129-ubuntu2404) \
              --build-arg max_jobs=16 \
              --build-arg USE_SCCACHE=1 \
              --build-arg GIT_REPO_CHECK=1 \
              --build-arg CUDA_VERSION=12.9.1 \
              --build-arg BUILD_BASE_IMAGE=pytorch/manylinux2_28-builder:cuda12.9-78e737ad29420ffc4800e677c51e2a852caf8359 \
              --build-arg UBUNTU_VERSION=24.04 \
              --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \
              --build-arg torch_cuda_arch_list="${CUDA_ARCH_X86_CU129}" \
              --build-arg INSTALL_KV_CONNECTORS=true \
              --target vllm-openai \
              --progress plain \
              -f docker/Dockerfile .
          - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404"
          - "docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404"
          - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404" release-images'

      - label: "Build release image - aarch64 - CUDA 12.9 - Ubuntu 24.04"
        depends_on: ~
        id: build-release-image-arm64-cuda-12-9-ubuntu2404
        agents:
          queue: arm64_cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - |
            DOCKER_BUILDKIT=1 docker build \
              $(bash .buildkite/scripts/docker-build-metadata-args.sh cu129-ubuntu2404) \
              --build-arg max_jobs=16 \
              --build-arg USE_SCCACHE=1 \
              --build-arg GIT_REPO_CHECK=1 \
              --build-arg CUDA_VERSION=12.9.1 \
              --build-arg BUILD_BASE_IMAGE=pytorch/manylinuxaarch64-builder:cuda12.9-78e737ad29420ffc4800e677c51e2a852caf8359 \
              --build-arg UBUNTU_VERSION=24.04 \
              --build-arg GDRCOPY_OS_VERSION=Ubuntu24_04 \
              --build-arg torch_cuda_arch_list="${CUDA_ARCH_AARCH64_CU129}" \
              --build-arg INSTALL_KV_CONNECTORS=true \
              --target vllm-openai \
              --progress plain \
              -f docker/Dockerfile .
          - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-cu129-ubuntu2404" release-images'

      - label: ":docker: Build release image - x86_64 - XPU"
        depends_on: ~
        id: build-xpu-release-image
        agents:
          queue: cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - |
            DOCKER_BUILDKIT=1 docker build \
              $(bash .buildkite/scripts/docker-build-metadata-args.sh xpu) \
              --build-arg GIT_REPO_CHECK=1 \
              --target vllm-openai \
              --progress plain \
              -f docker/Dockerfile.xpu .
          - "docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-xpu"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-$(uname -m)-xpu" release-images'
        env:
          DOCKER_BUILDKIT: "1"

      # CPU release images follow the CUDA pattern: a single build label per arch
      # pushes the arch-specific commit tag to ECR, consumed by both the nightly
      # DockerHub publish and the release publish step. The block auto-skips in
      # nightly (if NIGHTLY != "1") so the build runs unattended, mirroring
      # block-build-release-images.
      - block: "Build release image for x86_64 CPU"
        key: block-cpu-release-image-build
        depends_on: ~
        if: build.env("NIGHTLY") != "1"

      - label: "Build release image - x86_64 - CPU"
        key: build-cpu-release-image-x86
        depends_on:
          - block-cpu-release-image-build
        allow_dependency_failure: true
        agents:
          queue: cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_CPU_X86=true --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$BUILDKITE_COMMIT-x86_64 --progress plain --target vllm-openai -f docker/Dockerfile.cpu ."
          - "docker push public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$BUILDKITE_COMMIT-x86_64"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$BUILDKITE_COMMIT-x86_64" release-images'
        env:
          DOCKER_BUILDKIT: "1"

      - block: "Build release image for arm64 CPU"
        key: block-arm64-cpu-release-image-build
        depends_on: ~
        if: build.env("NIGHTLY") != "1"

      - label: "Build release image - arm64 - CPU"
        key: build-cpu-release-image-arm64
        depends_on:
          - block-arm64-cpu-release-image-build
        allow_dependency_failure: true
        agents:
          queue: arm64_cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --tag public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$BUILDKITE_COMMIT-arm64 --progress plain --target vllm-openai -f docker/Dockerfile.cpu ."
          - "docker push public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$BUILDKITE_COMMIT-arm64"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$BUILDKITE_COMMIT-arm64" release-images'
        env:
          DOCKER_BUILDKIT: "1"

  - group: "Publish release images"
    key: "publish-release-images"
    steps:
      - label: "Create multi-arch manifest - CUDA 13.0"
        depends_on:
          - build-release-image-x86
          - build-release-image-arm64
        id: create-multi-arch-manifest
        agents:
          queue: small_cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64 --amend"
          - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: CUDA 13.0" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT" release-manifests'

      - label: "Create multi-arch manifest - CUDA 12.9"
        depends_on:
          - build-release-image-x86-cuda-12-9
          - build-release-image-arm64-cuda-12-9
        id: create-multi-arch-manifest-cuda-12-9
        agents:
          queue: small_cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu129 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu129 --amend"
          - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: CUDA 12.9" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129" release-manifests'

      - label: "Create multi-arch manifest - Rubin CUDA 13.4"
        depends_on:
          - build-release-image-x86-rubin-cuda-13-4
          - build-release-image-arm64-rubin-cuda-13-4
        id: create-multi-arch-manifest-rubin-cuda-13-4
        if: build.env("NIGHTLY") == "1"
        agents:
          queue: small_cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu134 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu134 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu134 --amend"
          - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu134"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: Rubin CUDA 13.4" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu134" release-manifests'

      - label: "Create multi-arch manifest - CUDA 13.0 - Ubuntu 24.04"
        depends_on:
          - build-release-image-x86-ubuntu2404
          - build-release-image-arm64-ubuntu2404
        id: create-multi-arch-manifest-ubuntu2404
        agents:
          queue: small_cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-ubuntu2404 --amend"
          - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: CUDA 13.0 Ubuntu 24.04" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-ubuntu2404" release-manifests'

      - label: "Create multi-arch manifest - CUDA 12.9 - Ubuntu 24.04"
        depends_on:
          - build-release-image-x86-cuda-12-9-ubuntu2404
          - build-release-image-arm64-cuda-12-9-ubuntu2404
        id: create-multi-arch-manifest-cuda-12-9-ubuntu2404
        agents:
          queue: small_cpu_queue_release
        commands:
          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
          - "docker manifest create public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64-cu129-ubuntu2404 public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-aarch64-cu129-ubuntu2404 --amend"
          - "docker manifest push public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: CUDA 12.9 Ubuntu 24.04" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-cu129-ubuntu2404" release-manifests'

      - label: "Create manifest - XPU"
        depends_on:
          - build-xpu-release-image
        id: create-manifest-xpu
        agents:
          queue: small_cpu_queue_release
        commands:
          - "bash .buildkite/scripts/xpu/create-xpu-ecr-manifest.sh"
          - 'bash .buildkite/scripts/annotate-build-artifact.sh "Manifest: XPU" "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-xpu" release-manifests'

      - label: "Publish nightly multi-arch image to DockerHub"
        depends_on:
          - create-multi-arch-manifest
        if: build.env("NIGHTLY") == "1"
        agents:
          queue: small_cpu_queue_release
        commands:
          - "bash .buildkite/scripts/push-nightly-builds.sh"
          # Clean up old nightly builds (keep only last 14)
          - "bash .buildkite/scripts/cleanup-nightly-builds.sh"
        plugins:
          - docker-login#v3.0.0:
              username: vllmbot
              password-env: DOCKERHUB_TOKEN
        env:
          DOCKER_BUILDKIT: "1"
          DOCKERHUB_USERNAME: "vllmbot"

      - label: "Publish nightly multi-arch image to DockerHub - CUDA 12.9"
        depends_on:
          - create-multi-arch-manifest-cuda-12-9
        if: build.env("NIGHTLY") == "1"
        agents:
          queue: small_cpu_queue_release
        commands:
          - "bash .buildkite/scripts/push-nightly-builds.sh cu129"
          # Clean up old nightly builds (keep only last 14)
          - "bash .buildkite/scripts/cleanup-nightly-builds.sh cu129-nightly-"
        plugins:
          - docker-login#v3.0.0:
              username: vllmbot
              password-env: DOCKERHUB_TOKEN
        env:
          DOCKER_BUILDKIT: "1"
          DOCKERHUB_USERNAME: "vllmbot"

      - label: "Publish nightly multi-arch Rubin image to DockerHub - CUDA 13.4"
        depends_on:
          - create-multi-arch-manifest-rubin-cuda-13-4
        if: build.env("NIGHTLY") == "1"
        agents:
          queue: small_cpu_queue_release
        commands:
          - "bash .buildkite/scripts/push-nightly-builds.sh cu134"
          # Clean up old nightly builds (keep only last 14)
          - "bash .buildkite/scripts/cleanup-nightly-builds.sh cu134-nightly-"
        plugins:
          - docker-login#v3.0.0:
              username: vllmbot
              password-env: DOCKERHUB_TOKEN
        env:
          DOCKER_BUILDKIT: "1"
          DOCKERHUB_USERNAME: "vllmbot"

      - label: "Publish nightly CPU multi-arch image to DockerHub"
        depends_on:
          - build-cpu-release-image-x86
          - build-cpu-release-image-arm64
        if: build.env("NIGHTLY") == "1"
        agents:
          queue: small_cpu_queue_release
        commands:
          - "bash .buildkite/scripts/push-nightly-builds-cpu.sh"
          # Clean up old nightly builds (keep only last 14)
          - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly- vllm/vllm-openai-cpu"
        plugins:
          - docker-login#v3.0.0:
              username: vllmbot
              password-env: DOCKERHUB_TOKEN
        env:
          DOCKER_BUILDKIT: "1"
          DOCKERHUB_USERNAME: "vllmbot"

  - trigger: "perf-eval"
    label: "Trigger perf-eval (CUDA)"
    key: trigger-perf-eval-cuda
    if: build.env("NIGHTLY") == "1" || build.env("RUN_PERF_EVAL") == "1"
    depends_on:
      - build-release-image-x86
    async: true
    soft_fail: true
    build:
      branch: "main"
      message: "(NIGHTLY=${NIGHTLY:-0}) Evaluate vLLM $BUILDKITE_COMMIT (CUDA)"
      env:
        NIGHTLY: 1 # run all perf-eval workloads
        VLLM_COMMIT: "$BUILDKITE_COMMIT"
        VLLM_IMAGE_CUDA: "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-x86_64"

  - trigger: "perf-eval"
    label: "Trigger perf-eval (ROCm)"
    key: trigger-perf-eval-rocm
    if: build.env("NIGHTLY") == "1" || build.env("RUN_PERF_EVAL") == "1"
    depends_on:
      - build-rocm100-release-image
    async: true
    soft_fail: true
    build:
      branch: "main"
      message: "(NIGHTLY=${NIGHTLY:-0}) Evaluate vLLM $BUILDKITE_COMMIT (ROCm)"
      env:
        NIGHTLY: 1 # run all perf-eval workloads
        VLLM_COMMIT: "$BUILDKITE_COMMIT"
        VLLM_IMAGE_ROCM: "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-rocm100"

  # =============================================================================
  # ROCm Release Pipelines (x86_64 only)
  # =============================================================================
  #
  # One group per ROCm stack, identified by its base Dockerfile (ROCM_BASE_DOCKERFILE).
  # The scripts derive the ROCm version, wheel variant, cache keys and image tags from
  # it (.buildkite/scripts/rocm/stack.sh), so both groups share one set of step
  # definitions, anchored on the ROCm 10.0 group.
  #
  # - ROCm 10.0 (TheRock, docker/Dockerfile.rocm{,_base}) is the default: it owns the
  #   :nightly / :latest images and the root wheel index.
  # - ROCm 7.2 (docker/Dockerfile.rocm_72{,_base}) is kept for a transition period as
  #   rocm72 images and wheels.
  #
  # vLLM version is determined by the Buildkite checkout (like the CUDA pipeline).
  # =============================================================================

  - group: "Build ROCm 10.0 Wheel / Image"
    key: "build-rocm100"
    depends_on: ~
    steps:
      - label: ":rocm: Build ROCm 10.0 Base Image & Wheels"
        id: build-rocm100-base
        depends_on: ~
        agents: &rocm_build_agents
          queue: cpu_queue_release
        timeout_in_minutes: 360
        env: &rocm100_env
          ROCM_BASE_DOCKERFILE: "docker/Dockerfile.rocm_base"
          ROCM_DEFAULT_ARG: "--default"
          DOCKER_BUILDKIT: "1"
          S3_BUCKET: "vllm-wheels"
          DOCKERHUB_USERNAME: "vllmbot"
        commands: &rocm_base_commands
          - |
            set -euo pipefail
            source .buildkite/scripts/rocm/stack.sh "$${ROCM_BASE_DOCKERFILE}"
            SCCACHE_ARGS=(--build-arg USE_SCCACHE=1 --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache
              --build-arg SCCACHE_REGION_NAME=us-west-2 --build-arg SCCACHE_S3_NO_CREDENTIALS=0)
            aws ecr-public get-login-password --region us-east-1 | \
              docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7

            # Base image, cached in ECR by the base Dockerfile's hash
            if docker manifest inspect "$${ROCM_STACK_ECR_BASE}" > /dev/null 2>&1; then
              echo "ECR image cache HIT: $${ROCM_STACK_ECR_BASE}"
            else
              docker buildx build --file "$${ROCM_BASE_DOCKERFILE}" --tag "$${ROCM_STACK_ECR_BASE}" \
                "$${SCCACHE_ARGS[@]}" --push .
            fi

            # Base wheels, cached in S3 (see cache-rocm-base-wheels.sh)
            CACHE=.buildkite/scripts/cache-rocm-base-wheels.sh
            if [ "$$($${CACHE} "$${ROCM_BASE_DOCKERFILE}" check)" = "hit" ]; then
              $${CACHE} "$${ROCM_BASE_DOCKERFILE}" download
            else
              DEBS="rocm-base-debs:$${BUILDKITE_BUILD_NUMBER}-$${ROCM_STACK_VARIANT}"
              docker buildx build --file "$${ROCM_BASE_DOCKERFILE}" --tag "$${DEBS}" \
                --target debs_wheel_release "$${SCCACHE_ARGS[@]}" --load .
              mkdir -p "$${ROCM_STACK_BASE_WHEELS_DIR}"
              cid=$$(docker create "$${DEBS}")
              docker cp "$${cid}":/app/debs/. "$${ROCM_STACK_BASE_WHEELS_DIR}/"
              docker rm "$${cid}"
              $${CACHE} "$${ROCM_BASE_DOCKERFILE}" upload
            fi
            buildkite-agent artifact upload "$${ROCM_STACK_BASE_WHEELS_DIR}/*.whl"

      - label: ":python: Build vLLM ROCm 10.0 Wheel - x86_64"
        id: build-rocm100-vllm-wheel
        depends_on: build-rocm100-base
        agents: *rocm_build_agents
        timeout_in_minutes: 180
        env: *rocm100_env
        commands: &rocm_vllm_wheel_commands
          - |
            set -euo pipefail
            source .buildkite/scripts/rocm/stack.sh "$${ROCM_BASE_DOCKERFILE}"
            # Buildkite's default fetch doesn't update moved tags; version detection needs them
            git fetch --tags --force origin
            buildkite-agent artifact download "$${ROCM_STACK_BASE_WHEELS_DIR}/*.whl" .
            aws ecr-public get-login-password --region us-east-1 | \
              docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
            docker pull "$${ROCM_STACK_ECR_BASE}"
            mkdir -p docker/context/base-wheels
            touch docker/context/base-wheels/.keep
            cp "$${ROCM_STACK_BASE_WHEELS_DIR}"/*.whl docker/context/base-wheels/
            docker build --file "$${ROCM_STACK_DOCKERFILE}" --target export_vllm_wheel_release \
              --output type=local,dest=rocm-dist \
              --build-arg BASE_IMAGE="$${ROCM_STACK_ECR_BASE}" --build-arg REMOTE_VLLM=0 \
              --build-arg GIT_REPO_CHECK=1 --build-arg USE_SCCACHE=1 \
              --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
              --build-arg SCCACHE_REGION_NAME=us-west-2 --build-arg SCCACHE_S3_NO_CREDENTIALS=0 .
            mkdir -p "$${ROCM_STACK_VLLM_WHEEL_DIR}"
            cp rocm-dist/*.whl "$${ROCM_STACK_VLLM_WHEEL_DIR}/"
            buildkite-agent artifact upload "$${ROCM_STACK_VLLM_WHEEL_DIR}/*.whl"

      - label: ":s3: Upload ROCm 10.0 Wheels to S3"
        id: upload-rocm100-wheels
        depends_on: build-rocm100-vllm-wheel
        agents: *rocm_build_agents
        timeout_in_minutes: 60
        env: *rocm100_env
        commands: &rocm_upload_commands
          - |
            set -euo pipefail
            source .buildkite/scripts/rocm/stack.sh "$${ROCM_BASE_DOCKERFILE}"
            buildkite-agent artifact download "$${ROCM_STACK_BASE_WHEELS_DIR}/*.whl" .
            buildkite-agent artifact download "$${ROCM_STACK_VLLM_WHEEL_DIR}/*.whl" .
            bash .buildkite/scripts/upload-rocm-wheels.sh "$${ROCM_BASE_DOCKERFILE}"

      - label: ":memo: Annotate ROCm 10.0 wheel release"
        id: annotate-rocm100-release
        depends_on: upload-rocm100-wheels
        agents: *rocm_build_agents
        env: *rocm100_env
        commands: &rocm_annotate_commands
          - bash .buildkite/scripts/annotate-rocm-release.sh "$${ROCM_BASE_DOCKERFILE}" $${ROCM_DEFAULT_ARG}

      # Generate https://wheels.vllm.ai/rocm/ (the default stack's variant), so users can
      # `uv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm/` (release only)
      - block: "Generate Root Index for ROCm Wheels for Release"
        key: block-generate-root-index-rocm-wheels
        depends_on:
          - upload-rocm100-wheels
          - upload-rocm72-wheels

      - label: ":package: Generate Root Index for ROCm Wheels for Release"
        depends_on: block-generate-root-index-rocm-wheels
        id: generate-root-index-rocm-wheels
        agents: *rocm_build_agents
        env: *rocm100_env
        commands:
          - "bash tools/vllm-rocm/generate-rocm-wheels-root-index.sh"

      - label: ":docker: Build release image - x86_64 - ROCm 10.0"
        id: build-rocm100-release-image
        depends_on:
          - step: block-build-release-images
            allow_failure: true
          - step: build-rocm100-base
            allow_failure: false
        agents: *rocm_build_agents
        timeout_in_minutes: 180
        env: *rocm100_env
        commands: &rocm_release_image_commands
          - |
            set -euo pipefail
            source .buildkite/scripts/rocm/stack.sh "$${ROCM_BASE_DOCKERFILE}"
            aws ecr-public get-login-password --region us-east-1 | \
              docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
            docker pull "$${ROCM_STACK_ECR_BASE}"
            docker build --file "$${ROCM_STACK_DOCKERFILE}" --target vllm-openai \
              --tag "$${ROCM_STACK_ECR_IMAGE}" --progress plain \
              --build-arg max_jobs=16 --build-arg BASE_IMAGE="$${ROCM_STACK_ECR_BASE}" \
              --build-arg INSTALL_LMCACHE=true --build-arg USE_SCCACHE=1 \
              --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
              --build-arg SCCACHE_REGION_NAME=us-west-2 --build-arg SCCACHE_S3_NO_CREDENTIALS=0 .
            docker push "$${ROCM_STACK_ECR_IMAGE}"

      - label: "Publish nightly ROCm 10.0 image to DockerHub"
        id: publish-rocm100-nightly
        depends_on: build-rocm100-release-image
        if: build.env("NIGHTLY") == "1"
        agents: &rocm_publish_agents
          queue: small_cpu_queue_release
        env: *rocm100_env
        commands: &rocm_publish_nightly_commands
          - bash .buildkite/scripts/push-nightly-builds-rocm.sh "$${ROCM_BASE_DOCKERFILE}" $${ROCM_DEFAULT_ARG}
        plugins: &rocm_dockerhub_login
          - docker-login#v3.0.0:
              username: vllmbot
              password-env: DOCKERHUB_TOKEN

  - group: "Build ROCm 7.2 Wheel / Image"
    key: "build-rocm72"
    depends_on: ~
    steps:
      - label: ":rocm: Build ROCm 7.2 Base Image & Wheels"
        id: build-rocm72-base
        depends_on: ~
        agents: *rocm_build_agents
        timeout_in_minutes: 360
        env: &rocm72_env
          ROCM_BASE_DOCKERFILE: "docker/Dockerfile.rocm_72_base"
          ROCM_DEFAULT_ARG: ""
          DOCKER_BUILDKIT: "1"
          S3_BUCKET: "vllm-wheels"
          DOCKERHUB_USERNAME: "vllmbot"
        commands: *rocm_base_commands

      - label: ":python: Build vLLM ROCm 7.2 Wheel - x86_64"
        id: build-rocm72-vllm-wheel
        depends_on: build-rocm72-base
        agents: *rocm_build_agents
        timeout_in_minutes: 180
        env: *rocm72_env
        commands: *rocm_vllm_wheel_commands

      - label: ":s3: Upload ROCm 7.2 Wheels to S3"
        id: upload-rocm72-wheels
        depends_on: build-rocm72-vllm-wheel
        agents: *rocm_build_agents
        timeout_in_minutes: 60
        env: *rocm72_env
        commands: *rocm_upload_commands

      - label: ":memo: Annotate ROCm 7.2 wheel release"
        id: annotate-rocm72-release
        depends_on: upload-rocm72-wheels
        agents: *rocm_build_agents
        env: *rocm72_env
        commands: *rocm_annotate_commands

      - label: ":docker: Build release image - x86_64 - ROCm 7.2"
        id: build-rocm72-release-image
        depends_on:
          - step: block-build-release-images
            allow_failure: true
          - step: build-rocm72-base
            allow_failure: false
        agents: *rocm_build_agents
        timeout_in_minutes: 180
        env: *rocm72_env
        commands: *rocm_release_image_commands

      - label: "Publish nightly ROCm 7.2 image to DockerHub"
        id: publish-rocm72-nightly
        depends_on: build-rocm72-release-image
        if: build.env("NIGHTLY") == "1"
        agents: *rocm_publish_agents
        env: *rocm72_env
        commands: *rocm_publish_nightly_commands
        plugins: *rocm_dockerhub_login

      - label: "Publish nightly XPU image to DockerHub"
        depends_on:
          - create-manifest-xpu
        if: build.env("NIGHTLY") == "1"
        agents:
          queue: small_cpu_queue_release
        commands:
          - "bash .buildkite/scripts/xpu/push-nightly-builds-xpu.sh"
          - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly- vllm/vllm-openai-xpu"
        plugins:
          - docker-login#v3.0.0:
              username: vllmbot
              password-env: DOCKERHUB_TOKEN
        env:
          DOCKER_BUILDKIT: "1"
          DOCKERHUB_USERNAME: "vllmbot"

  # =============================================================================
  # Publish to DockerHub and PyPI (at the end so all builds complete first)
  # =============================================================================

  - block: "Publish release images to DockerHub"
    key: block-publish-release-images
    depends_on:
      - create-multi-arch-manifest
      - create-multi-arch-manifest-cuda-12-9
      - create-multi-arch-manifest-ubuntu2404
      - create-multi-arch-manifest-cuda-12-9-ubuntu2404
      - create-manifest-xpu
      - build-rocm100-release-image
      - build-rocm72-release-image
      - input-release-version
      # Wait for CPU builds if their block steps were unblocked, so publish
      # doesn't race the in-progress CPU build. allow_failure lets publish
      # proceed when the operator legitimately leaves the CPU block steps
      # unblocked or the CPU build fails.
      - step: build-cpu-release-image-x86
        allow_failure: true
      - step: build-cpu-release-image-arm64
        allow_failure: true

  - label: "Publish {{matrix}} release images to DockerHub"
    depends_on:
      - block-publish-release-images
    key: publish-release-images-dockerhub
    agents:
      queue: small_cpu_queue_release
    commands:
      - "bash .buildkite/scripts/publish-release-images.sh {{matrix}}"
    matrix:
      - "cuda-13-0"
      - "cuda-12-9"
      - "cuda-13-0-ubuntu-24-04"
      - "cuda-12-9-ubuntu-24-04"
      - "rocm"
      - "rocm72"
      - "xpu"
      - "cpu"
    plugins:
      - docker-login#v3.0.0:
          username: vllmbot
          password-env: DOCKERHUB_TOKEN
    env:
      DOCKER_BUILDKIT: "1"
      DOCKERHUB_USERNAME: "vllmbot"

  - group: "Publish wheels"
    key: "publish-wheels"
    steps:
      - block: "Confirm update release wheels to PyPI (experimental, use with caution)?"
        key: block-upload-release-wheels
        depends_on:
          - input-release-version
          - build-wheels
          - build-additional-wheels
          - generate-additional-wheel-indices

      - label: "Upload release wheels to PyPI"
        depends_on:
          - block-upload-release-wheels
        id: upload-release-wheels
        agents:
          queue: small_cpu_queue_release
        commands:
          - "bash .buildkite/scripts/upload-release-wheels-pypi.sh"