[ROCm][CI] Build and publish TheRock nightly docker images (#55014)

Signed-off-by: Rohan Potdar <rohan.potdar@amd.com>
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Rohan Potdar
2026-09-03 21:34:22 -05:00
committed by GitHub
co-authored by Claude Opus 5 Andreas Karatzas OpenAI Codex
parent d9e2b5238d
commit 7dc30f5a66
2 changed files with 219 additions and 34 deletions
+138
View File
@@ -959,6 +959,144 @@ steps:
DOCKER_BUILDKIT: "1"
DOCKERHUB_USERNAME: "vllmbot"
# =============================================================================
# ROCk Release Pipeline (x86_64 only)
# =============================================================================
#
# Same shape as the ROCm pipeline above (Jobs 1 and 6 only), but built from
# docker/Dockerfile.rock_base and docker/Dockerfile.rock, which install the
# ROCm SDK from TheRock wheels instead of the ROCm apt repos.
#
# Note: ROCm SDK version is determined by ROCM_SDK_VERSION in
# docker/Dockerfile.rock_base
#
# =============================================================================
- group: "Build ROCk Image"
key: "build-rock-image"
depends_on: ~
steps:
# ROCk Job 1: Build ROCk Base Image (with ECR caching)
- label: ":rocm: Build ROCk Base Image"
id: build-rock-base-image
depends_on: ~
agents:
queue: cpu_queue_release
timeout_in_minutes: 360
commands:
- |
set -euo pipefail
# The SDK version, torch wheels and component branches all live in
# the Dockerfile, so its hash is the cache key.
CACHE_KEY=$$(sha256sum docker/Dockerfile.rock_base | cut -c1-16)
ECR_CACHE_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:$${CACHE_KEY}-rock-base"
echo "========================================"
echo "ROCk Base Build Configuration"
echo "========================================"
echo " CACHE_KEY: $${CACHE_KEY}"
echo " ECR_CACHE_TAG: $${ECR_CACHE_TAG}"
echo "========================================"
# Login to ECR
aws ecr-public get-login-password --region us-east-1 | \
docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
if docker manifest inspect "$${ECR_CACHE_TAG}" > /dev/null 2>&1; then
echo "ECR image cache HIT"
else
echo "CACHE MISS - Building from scratch..."
DOCKER_BUILDKIT=1 docker buildx build \
--file docker/Dockerfile.rock_base \
--tag "$${ECR_CACHE_TAG}" \
--build-arg USE_SCCACHE=1 \
--build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
--build-arg SCCACHE_REGION_NAME=us-west-2 \
--build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
--push \
.
fi
# Save ECR tag for downstream jobs
buildkite-agent meta-data set "rock-base-image-tag" "$${ECR_CACHE_TAG}"
env:
DOCKER_BUILDKIT: "1"
# ROCk Job 2: Build ROCk Release Docker Image
- label: ":docker: Build release image - x86_64 - ROCk"
id: build-rock-release-image
depends_on:
- step: block-build-release-images
allow_failure: true
- step: build-rock-base-image
allow_failure: false
agents:
queue: cpu_queue_release
timeout_in_minutes: 60
commands:
- |
set -euo pipefail
# Login to ECR
aws ecr-public get-login-password --region us-east-1 | \
docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rock-base-image-tag 2>/dev/null || echo '')"
if [ -z "$${ECR_IMAGE_TAG}" ]; then
echo "ERROR: rock-base-image-tag metadata not found"
echo "This should have been set by the build-rock-base-image job"
exit 1
fi
docker pull "$${ECR_IMAGE_TAG}"
# Pass the base image ECR tag to downstream steps (nightly publish)
buildkite-agent meta-data set "rock-base-ecr-tag" "$${ECR_IMAGE_TAG}"
echo "========================================"
echo "Building vLLM ROCk release image with:"
echo " BASE_IMAGE: $${ECR_IMAGE_TAG}"
echo " BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}"
echo "========================================"
DOCKER_BUILDKIT=1 docker build \
--build-arg max_jobs=16 \
--build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \
--build-arg USE_SCCACHE=1 \
--build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
--build-arg SCCACHE_REGION_NAME=us-west-2 \
--build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
--build-arg INSTALL_LMCACHE=true \
--tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rock \
--target vllm-openai \
--progress plain \
-f docker/Dockerfile.rock .
docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rock
env:
DOCKER_BUILDKIT: "1"
- label: "Publish nightly ROCk image to DockerHub"
depends_on:
- build-rock-release-image
if: build.env("NIGHTLY") == "1"
agents:
queue: small_cpu_queue_release
commands:
- "bash .buildkite/scripts/push-nightly-builds-rocm.sh rock rocm714"
# Clean up old nightly builds (keep only last 14)
- "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly-rocm714- vllm/vllm-openai-rocm"
- "bash .buildkite/scripts/cleanup-nightly-builds.sh base-nightly-rocm714- vllm/vllm-openai-rocm"
plugins:
- docker-login#v3.0.0:
username: vllmbot
password-env: DOCKERHUB_TOKEN
env:
DOCKER_BUILDKIT: "1"
DOCKERHUB_USERNAME: "vllmbot"
# =============================================================================
# Publish to DockerHub and PyPI (at the end so all builds complete first)
# =============================================================================
+81 -34
View File
@@ -2,61 +2,108 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Push ROCm nightly base image and nightly image from ECR
# to Docker Hub as vllm/vllm-openai-rocm:base-nightly and vllm/vllm-openai-rocm:nightly
# and vllm/vllm-openai-rocm:base-nightly-<commit> and vllm/vllm-openai-rocm:nightly-<commit>.
# Run when NIGHTLY=1 after build-rocm-release-image has pushed to ECR.
# Push a ROCm-family nightly base image and nightly image from ECR to Docker Hub
# under vllm/vllm-openai-rocm, as base-nightly[-<variant>], nightly[-<variant>]
# and their -<commit> forms.
# Run when NIGHTLY=1 after the matching build-*-release-image step has pushed to ECR.
#
# Usage: push-nightly-builds-rocm.sh [ECR_TAG_SUFFIX TAG_VARIANT]
# Pass both arguments or neither. With none, the default ROCm apt build is
# published as :nightly. The TheRock build passes "rock" (the source ECR tag
# suffix) and "rocm714" (the Docker Hub tag flavor), giving :nightly-rocm714.
#
# Local testing (no push to Docker Hub):
# BUILDKITE_COMMIT=<commit-with-rocm-image-in-ecr> DRY_RUN=1 bash .buildkite/scripts/push-nightly-builds-rocm.sh
# BASE_ECR_IMAGE=<full-base-image-reference> \
# BUILDKITE_COMMIT=<commit-with-image-in-ecr> DRY_RUN=1 \
# bash .buildkite/scripts/push-nightly-builds-rocm.sh
# Requires: AWS CLI configured (for ECR public login), Docker. For full run: Docker Hub login.
set -ex
set -euxo pipefail
# Use BUILDKITE_COMMIT from env (required; set to a commit that has ROCm image in ECR for local test)
BUILDKITE_COMMIT="${BUILDKITE_COMMIT:?Set BUILDKITE_COMMIT to the commit SHA that has the ROCm image in ECR (e.g. from a previous release pipeline run)}"
usage() {
echo "Usage: $0 [ECR_TAG_SUFFIX TAG_VARIANT]" >&2
}
DOCKERHUB_REPO="vllm/vllm-openai-rocm"
case "$#" in
0)
ECR_TAG_SUFFIX="rocm"
TAG_VARIANT=""
;;
2)
if [[ -z "$1" || -z "$2" ]]; then
usage
exit 2
fi
ECR_TAG_SUFFIX="$1"
TAG_VARIANT="$2"
;;
*)
usage
exit 2
;;
esac
# Use BUILDKITE_COMMIT from env (required; set to a commit that has the image in ECR for local test)
BUILDKITE_COMMIT="${BUILDKITE_COMMIT:?Set BUILDKITE_COMMIT to the commit SHA that has the image in ECR (e.g. from a previous release pipeline run)}"
DRY_RUN="${DRY_RUN:-0}"
# Get the base image ECR tag (set by build-rocm-release-image pipeline step)
BASE_ORIG_TAG="$(buildkite-agent meta-data get rocm-base-ecr-tag 2>/dev/null || echo "")"
if [ -z "$BASE_ORIG_TAG" ]; then
echo "WARNING: rocm-base-ecr-tag metadata not found, falling back to commit-based tag"
BASE_ORIG_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm-base"
ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-release-repo"
# Get the base image ECR tag (set by the build-*-release-image pipeline step)
BASE_METADATA_KEY="${ECR_TAG_SUFFIX}-base-ecr-tag"
if [[ -n "${BASE_ECR_IMAGE:-}" ]]; then
if [[ "$DRY_RUN" != "1" ]]; then
echo "ERROR: BASE_ECR_IMAGE may only be used with DRY_RUN=1" >&2
exit 1
fi
BASE_ORIG_TAG="$BASE_ECR_IMAGE"
elif ! BASE_ORIG_TAG="$(buildkite-agent meta-data get "$BASE_METADATA_KEY")"; then
echo "ERROR: Failed to read required Buildkite metadata '$BASE_METADATA_KEY'" >&2
exit 1
fi
if [[ -z "$BASE_ORIG_TAG" ]]; then
echo "ERROR: Required Buildkite metadata '$BASE_METADATA_KEY' is empty" >&2
echo "Set BASE_ECR_IMAGE to the full ECR base image reference for local testing" >&2
exit 1
fi
ORIG_TAG="${BUILDKITE_COMMIT}-rocm"
BASE_TAG_NAME="base-nightly"
TAG_NAME="nightly"
BASE_TAG_NAME_COMMIT="base-nightly-${BUILDKITE_COMMIT}"
TAG_NAME_COMMIT="nightly-${BUILDKITE_COMMIT}"
ORIG_TAG="${BUILDKITE_COMMIT}-${ECR_TAG_SUFFIX}"
VARIANT_SUFFIX=""
[[ -n "$TAG_VARIANT" ]] && VARIANT_SUFFIX="-$TAG_VARIANT"
BASE_TAG_NAME="base-nightly${VARIANT_SUFFIX}"
TAG_NAME="nightly${VARIANT_SUFFIX}"
BASE_TAG_NAME_COMMIT="base-nightly${VARIANT_SUFFIX}-${BUILDKITE_COMMIT}"
TAG_NAME_COMMIT="nightly${VARIANT_SUFFIX}-${BUILDKITE_COMMIT}"
echo "Pushing ROCm base image from ECR: $BASE_ORIG_TAG"
echo "Pushing ROCm release image from ECR tag: $ORIG_TAG to Docker Hub as $TAG_NAME and $TAG_NAME_COMMIT"
echo "Pushing base image from ECR: $BASE_ORIG_TAG"
echo "Pushing release image from ECR tag: $ORIG_TAG to Docker Hub as $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT"
[[ "$DRY_RUN" == "1" ]] && echo "[DRY_RUN] Skipping push to Docker Hub"
# Login to ECR and pull the image built by build-rocm-release-image
# Login to ECR and pull the images built by build-*-release-image
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
docker pull "$BASE_ORIG_TAG"
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG"
docker pull "${ECR_REPO}:${ORIG_TAG}"
# Tag for Docker Hub (base-nightly and base-nightly-<commit>, nightly and nightly-<commit>)
docker tag "$BASE_ORIG_TAG" vllm/vllm-openai-rocm:"$BASE_TAG_NAME"
docker tag "$BASE_ORIG_TAG" vllm/vllm-openai-rocm:"$BASE_TAG_NAME_COMMIT"
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG" vllm/vllm-openai-rocm:"$TAG_NAME"
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG" vllm/vllm-openai-rocm:"$TAG_NAME_COMMIT"
docker tag "$BASE_ORIG_TAG" "$DOCKERHUB_REPO":"$BASE_TAG_NAME"
docker tag "$BASE_ORIG_TAG" "$DOCKERHUB_REPO":"$BASE_TAG_NAME_COMMIT"
docker tag "${ECR_REPO}:${ORIG_TAG}" "$DOCKERHUB_REPO":"$TAG_NAME"
docker tag "${ECR_REPO}:${ORIG_TAG}" "$DOCKERHUB_REPO":"$TAG_NAME_COMMIT"
if [[ "$DRY_RUN" == "1" ]]; then
echo "[DRY_RUN] Would push vllm/vllm-openai-rocm:$BASE_TAG_NAME and vllm/vllm-openai-rocm:$BASE_TAG_NAME_COMMIT"
echo "[DRY_RUN] Would push vllm/vllm-openai-rocm:$TAG_NAME and vllm/vllm-openai-rocm:$TAG_NAME_COMMIT"
echo "[DRY_RUN] Would push $DOCKERHUB_REPO:$BASE_TAG_NAME and $DOCKERHUB_REPO:$BASE_TAG_NAME_COMMIT"
echo "[DRY_RUN] Would push $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT"
echo "[DRY_RUN] Local tags created. Exiting without push."
exit 0
fi
# Push to Docker Hub (docker-login plugin runs before this step in CI)
docker push vllm/vllm-openai-rocm:"$BASE_TAG_NAME"
docker push vllm/vllm-openai-rocm:"$BASE_TAG_NAME_COMMIT"
docker push vllm/vllm-openai-rocm:"$TAG_NAME"
docker push vllm/vllm-openai-rocm:"$TAG_NAME_COMMIT"
docker push "$DOCKERHUB_REPO":"$BASE_TAG_NAME"
docker push "$DOCKERHUB_REPO":"$BASE_TAG_NAME_COMMIT"
docker push "$DOCKERHUB_REPO":"$TAG_NAME"
docker push "$DOCKERHUB_REPO":"$TAG_NAME_COMMIT"
echo "Pushed vllm/vllm-openai-rocm:$BASE_TAG_NAME and vllm/vllm-openai-rocm:$BASE_TAG_NAME_COMMIT"
echo "Pushed vllm/vllm-openai-rocm:$TAG_NAME and vllm/vllm-openai-rocm:$TAG_NAME_COMMIT"
echo "Pushed $DOCKERHUB_REPO:$BASE_TAG_NAME and $DOCKERHUB_REPO:$BASE_TAG_NAME_COMMIT"
echo "Pushed $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT"