From 299d2b565517044847f7300c11b9eb4c41931c36 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Thu, 9 Jul 2026 07:02:40 -0700 Subject: [PATCH] [CI] Annotate built Docker image tags on the Buildkite build page (#48101) Signed-off-by: khluu Co-authored-by: Claude Opus 4.8 --- .buildkite/image_build/image_build.sh | 8 ++++ .buildkite/image_build/image_build_arm64.sh | 38 +++++++++---------- .buildkite/image_build/image_build_cpu.sh | 30 +++++++-------- .../image_build/image_build_cpu_arm64.sh | 28 +++++++------- .buildkite/image_build/image_build_hpu.sh | 30 +++++++-------- .../image_build/image_build_torch_nightly.sh | 3 ++ .buildkite/image_build/image_build_xpu.sh | 28 +++++++------- .buildkite/scripts/annotate-image-build.sh | 36 ++++++++++++++++++ 8 files changed, 124 insertions(+), 77 deletions(-) create mode 100755 .buildkite/scripts/annotate-image-build.sh diff --git a/.buildkite/image_build/image_build.sh b/.buildkite/image_build/image_build.sh index 10c03c3e17..6c8d0b9b36 100755 --- a/.buildkite/image_build/image_build.sh +++ b/.buildkite/image_build/image_build.sh @@ -79,12 +79,18 @@ setup_buildx_builder() { docker buildx ls | grep -E '^\*|^NAME' || docker buildx ls } +annotate_image_tags() { + .buildkite/scripts/annotate-image-build.sh \ + "${IMAGE_TAG:-}" "${IMAGE_TAG_LATEST:-}" +} + check_and_skip_if_image_exists() { if [[ -n "${IMAGE_TAG:-}" ]]; then echo "--- :mag: Checking if image exists" if docker manifest inspect "${IMAGE_TAG}" >/dev/null 2>&1; then echo "Image already exists: ${IMAGE_TAG}" echo "Skipping build" + annotate_image_tags exit 0 fi echo "Image not found, proceeding with build" @@ -254,3 +260,5 @@ echo "--- :docker: Building ${TARGET}" docker --debug buildx bake -f "${VLLM_BAKE_FILE_PATH}" -f "${CI_HCL_PATH}" --progress plain "${TARGET}" echo "--- :white_check_mark: Build complete" + +annotate_image_tags diff --git a/.buildkite/image_build/image_build_arm64.sh b/.buildkite/image_build/image_build_arm64.sh index 0280d31c0c..3f73987846 100755 --- a/.buildkite/image_build/image_build_arm64.sh +++ b/.buildkite/image_build/image_build_arm64.sh @@ -9,30 +9,30 @@ fi REGISTRY=$1 REPO=$2 BUILDKITE_COMMIT=$3 +IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64" # authenticate with AWS ECR aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true # skip build if image already exists -if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64) ]]; then - echo "Image not found, proceeding with build..." -else +if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then echo "Image found" - exit 0 +else + echo "Image not found, proceeding with build..." + # build for arm64 GPU targets: Grace/GH200 (sm_90) and DGX Spark/GB10 + # (sm_121, family-covered by 12.0 under CUDA 13) + docker build --file docker/Dockerfile \ + --platform linux/arm64 \ + --build-arg max_jobs=16 \ + --build-arg nvcc_threads=4 \ + --build-arg torch_cuda_arch_list="9.0 12.0" \ + --build-arg USE_SCCACHE=1 \ + --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ + --tag "$IMAGE" \ + --target test \ + --progress plain . + # push + docker push "$IMAGE" fi -# build for arm64 GPU targets: Grace/GH200 (sm_90) and DGX Spark/GB10 -# (sm_121, family-covered by 12.0 under CUDA 13) -docker build --file docker/Dockerfile \ - --platform linux/arm64 \ - --build-arg max_jobs=16 \ - --build-arg nvcc_threads=4 \ - --build-arg torch_cuda_arch_list="9.0 12.0" \ - --build-arg USE_SCCACHE=1 \ - --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ - --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64 \ - --target test \ - --progress plain . - -# push -docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64 +.buildkite/scripts/annotate-image-build.sh "$IMAGE" diff --git a/.buildkite/image_build/image_build_cpu.sh b/.buildkite/image_build/image_build_cpu.sh index 035f070ab8..3ac1a110f6 100755 --- a/.buildkite/image_build/image_build_cpu.sh +++ b/.buildkite/image_build/image_build_cpu.sh @@ -9,26 +9,26 @@ fi REGISTRY=$1 REPO=$2 BUILDKITE_COMMIT=$3 +IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-cpu" # authenticate with AWS ECR aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true # skip build if image already exists -if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu) ]]; then - echo "Image not found, proceeding with build..." -else +if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then echo "Image found" - exit 0 +else + echo "Image not found, proceeding with build..." + # build + docker build --file docker/Dockerfile.cpu \ + --build-arg max_jobs=16 \ + --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ + --build-arg VLLM_CPU_X86=true \ + --tag "$IMAGE" \ + --target vllm-test \ + --progress plain . + # push + docker push "$IMAGE" fi -# build -docker build --file docker/Dockerfile.cpu \ - --build-arg max_jobs=16 \ - --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ - --build-arg VLLM_CPU_X86=true \ - --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu \ - --target vllm-test \ - --progress plain . - -# push -docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-cpu +.buildkite/scripts/annotate-image-build.sh "$IMAGE" diff --git a/.buildkite/image_build/image_build_cpu_arm64.sh b/.buildkite/image_build/image_build_cpu_arm64.sh index b561e2c2e4..7bb4f10f72 100755 --- a/.buildkite/image_build/image_build_cpu_arm64.sh +++ b/.buildkite/image_build/image_build_cpu_arm64.sh @@ -9,25 +9,25 @@ fi REGISTRY=$1 REPO=$2 BUILDKITE_COMMIT=$3 +IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-arm64-cpu" # authenticate with AWS ECR aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true # skip build if image already exists -if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64-cpu) ]]; then - echo "Image not found, proceeding with build..." -else +if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then echo "Image found" - exit 0 +else + echo "Image not found, proceeding with build..." + # build + docker build --file docker/Dockerfile.cpu \ + --build-arg max_jobs=16 \ + --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ + --tag "$IMAGE" \ + --target vllm-test \ + --progress plain . + # push + docker push "$IMAGE" fi -# build -docker build --file docker/Dockerfile.cpu \ - --build-arg max_jobs=16 \ - --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ - --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64-cpu \ - --target vllm-test \ - --progress plain . - -# push -docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64-cpu +.buildkite/scripts/annotate-image-build.sh "$IMAGE" diff --git a/.buildkite/image_build/image_build_hpu.sh b/.buildkite/image_build/image_build_hpu.sh index df900dc603..35b02c7ec9 100755 --- a/.buildkite/image_build/image_build_hpu.sh +++ b/.buildkite/image_build/image_build_hpu.sh @@ -9,26 +9,26 @@ fi REGISTRY=$1 REPO=$2 BUILDKITE_COMMIT=$3 +IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-hpu" # authenticate with AWS ECR aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true # skip build if image already exists -if [[ -z $(docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu) ]]; then - echo "Image not found, proceeding with build..." -else +if docker manifest inspect "$IMAGE" >/dev/null 2>&1; then echo "Image found" - exit 0 +else + echo "Image not found, proceeding with build..." + # build + docker build \ + --file tests/pytorch_ci_hud_benchmark/Dockerfile.hpu \ + --build-arg max_jobs=16 \ + --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ + --tag "$IMAGE" \ + --progress plain \ + https://github.com/vllm-project/vllm-gaudi.git + # push + docker push "$IMAGE" fi -# build -docker build \ - --file tests/pytorch_ci_hud_benchmark/Dockerfile.hpu \ - --build-arg max_jobs=16 \ - --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ - --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu \ - --progress plain \ - https://github.com/vllm-project/vllm-gaudi.git - -# push -docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-hpu +.buildkite/scripts/annotate-image-build.sh "$IMAGE" diff --git a/.buildkite/image_build/image_build_torch_nightly.sh b/.buildkite/image_build/image_build_torch_nightly.sh index cbd08aa7bd..d13b3351d3 100755 --- a/.buildkite/image_build/image_build_torch_nightly.sh +++ b/.buildkite/image_build/image_build_torch_nightly.sh @@ -40,6 +40,7 @@ docker buildx ls echo "--- :mag: Checking if image already exists" if docker manifest inspect "$IMAGE_TAG" >/dev/null 2>&1; then echo "Image found: $IMAGE_TAG — skipping build" + .buildkite/scripts/annotate-image-build.sh "$IMAGE_TAG" exit 0 fi echo "Image not found, proceeding with build..." @@ -66,3 +67,5 @@ docker buildx build --file docker/Dockerfile \ --progress plain . echo "--- :white_check_mark: Torch nightly image build complete: $IMAGE_TAG" + +.buildkite/scripts/annotate-image-build.sh "$IMAGE_TAG" diff --git a/.buildkite/image_build/image_build_xpu.sh b/.buildkite/image_build/image_build_xpu.sh index 45417b7339..adc544a233 100755 --- a/.buildkite/image_build/image_build_xpu.sh +++ b/.buildkite/image_build/image_build_xpu.sh @@ -9,26 +9,26 @@ fi REGISTRY=$1 REPO=$2 BUILDKITE_COMMIT=$3 +IMAGE="$REGISTRY/$REPO:$BUILDKITE_COMMIT-xpu" # authenticate with AWS ECR aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$REGISTRY" || true aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin 936637512419.dkr.ecr.us-east-1.amazonaws.com || true # skip build if image already exists -if ! docker manifest inspect "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-xpu &> /dev/null; then - echo "Image not found, proceeding with build..." -else +if docker manifest inspect "$IMAGE" &> /dev/null; then echo "Image found" - exit 0 +else + echo "Image not found, proceeding with build..." + # build + docker build \ + --file docker/Dockerfile.xpu \ + --build-arg max_jobs=16 \ + --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ + --tag "$IMAGE" \ + --progress plain . + # push + docker push "$IMAGE" fi -# build -docker build \ - --file docker/Dockerfile.xpu \ - --build-arg max_jobs=16 \ - --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ - --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-xpu \ - --progress plain . - -# push -docker push "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-xpu +.buildkite/scripts/annotate-image-build.sh "$IMAGE" diff --git a/.buildkite/scripts/annotate-image-build.sh b/.buildkite/scripts/annotate-image-build.sh new file mode 100755 index 0000000000..174d88bf67 --- /dev/null +++ b/.buildkite/scripts/annotate-image-build.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# +# Append the Docker image tag(s) an image-build step pushed to a Buildkite +# annotation, so the built image tags show up on the build page instead of +# being buried in the job logs. +# +# Usage: annotate-image-build.sh [ ...] +set -euo pipefail + +# buildkite-agent only exists on Buildkite agents; no-op elsewhere so the +# image build scripts stay runnable locally. +if ! command -v buildkite-agent >/dev/null 2>&1; then + echo "buildkite-agent not found; skipping image tag annotation" + exit 0 +fi + +label="${BUILDKITE_LABEL:-Image build}" +content="" +for image in "$@"; do + [[ -n "$image" ]] || continue + content+="- **${label}**: \`${image}\`"$'\n' +done + +if [[ -z "$content" ]]; then + echo "No image tags provided; nothing to annotate" + exit 0 +fi + +# Best-effort: a flaky annotation must never fail an otherwise successful +# (and expensive) image build. +if ! printf '%s' "$content" | \ + buildkite-agent annotate --append --style 'info' --context 'docker-images'; then + echo "warning: failed to annotate build with image tags" +fi