[ROCm][CI] Build and publish TheRock nightly docker images (#55014)
Signed-off-by: Rohan Potdar <rohan.potdar@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude Opus 5
Andreas Karatzas
OpenAI Codex
parent
d9e2b5238d
commit
7dc30f5a66
@@ -959,6 +959,144 @@ steps:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
DOCKERHUB_USERNAME: "vllmbot"
|
||||
|
||||
# =============================================================================
|
||||
# ROCk Release Pipeline (x86_64 only)
|
||||
# =============================================================================
|
||||
#
|
||||
# Same shape as the ROCm pipeline above (Jobs 1 and 6 only), but built from
|
||||
# docker/Dockerfile.rock_base and docker/Dockerfile.rock, which install the
|
||||
# ROCm SDK from TheRock wheels instead of the ROCm apt repos.
|
||||
#
|
||||
# Note: ROCm SDK version is determined by ROCM_SDK_VERSION in
|
||||
# docker/Dockerfile.rock_base
|
||||
#
|
||||
# =============================================================================
|
||||
|
||||
- group: "Build ROCk Image"
|
||||
key: "build-rock-image"
|
||||
depends_on: ~
|
||||
steps:
|
||||
# ROCk Job 1: Build ROCk Base Image (with ECR caching)
|
||||
- label: ":rocm: Build ROCk Base Image"
|
||||
id: build-rock-base-image
|
||||
depends_on: ~
|
||||
agents:
|
||||
queue: cpu_queue_release
|
||||
timeout_in_minutes: 360
|
||||
commands:
|
||||
- |
|
||||
set -euo pipefail
|
||||
|
||||
# The SDK version, torch wheels and component branches all live in
|
||||
# the Dockerfile, so its hash is the cache key.
|
||||
CACHE_KEY=$$(sha256sum docker/Dockerfile.rock_base | cut -c1-16)
|
||||
ECR_CACHE_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:$${CACHE_KEY}-rock-base"
|
||||
|
||||
echo "========================================"
|
||||
echo "ROCk Base Build Configuration"
|
||||
echo "========================================"
|
||||
echo " CACHE_KEY: $${CACHE_KEY}"
|
||||
echo " ECR_CACHE_TAG: $${ECR_CACHE_TAG}"
|
||||
echo "========================================"
|
||||
|
||||
# Login to ECR
|
||||
aws ecr-public get-login-password --region us-east-1 | \
|
||||
docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
|
||||
|
||||
if docker manifest inspect "$${ECR_CACHE_TAG}" > /dev/null 2>&1; then
|
||||
echo "ECR image cache HIT"
|
||||
else
|
||||
echo "CACHE MISS - Building from scratch..."
|
||||
|
||||
DOCKER_BUILDKIT=1 docker buildx build \
|
||||
--file docker/Dockerfile.rock_base \
|
||||
--tag "$${ECR_CACHE_TAG}" \
|
||||
--build-arg USE_SCCACHE=1 \
|
||||
--build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
|
||||
--build-arg SCCACHE_REGION_NAME=us-west-2 \
|
||||
--build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
|
||||
--push \
|
||||
.
|
||||
fi
|
||||
|
||||
# Save ECR tag for downstream jobs
|
||||
buildkite-agent meta-data set "rock-base-image-tag" "$${ECR_CACHE_TAG}"
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
|
||||
# ROCk Job 2: Build ROCk Release Docker Image
|
||||
- label: ":docker: Build release image - x86_64 - ROCk"
|
||||
id: build-rock-release-image
|
||||
depends_on:
|
||||
- step: block-build-release-images
|
||||
allow_failure: true
|
||||
- step: build-rock-base-image
|
||||
allow_failure: false
|
||||
agents:
|
||||
queue: cpu_queue_release
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- |
|
||||
set -euo pipefail
|
||||
|
||||
# Login to ECR
|
||||
aws ecr-public get-login-password --region us-east-1 | \
|
||||
docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
|
||||
|
||||
ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rock-base-image-tag 2>/dev/null || echo '')"
|
||||
if [ -z "$${ECR_IMAGE_TAG}" ]; then
|
||||
echo "ERROR: rock-base-image-tag metadata not found"
|
||||
echo "This should have been set by the build-rock-base-image job"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
docker pull "$${ECR_IMAGE_TAG}"
|
||||
|
||||
# Pass the base image ECR tag to downstream steps (nightly publish)
|
||||
buildkite-agent meta-data set "rock-base-ecr-tag" "$${ECR_IMAGE_TAG}"
|
||||
|
||||
echo "========================================"
|
||||
echo "Building vLLM ROCk release image with:"
|
||||
echo " BASE_IMAGE: $${ECR_IMAGE_TAG}"
|
||||
echo " BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}"
|
||||
echo "========================================"
|
||||
|
||||
DOCKER_BUILDKIT=1 docker build \
|
||||
--build-arg max_jobs=16 \
|
||||
--build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \
|
||||
--build-arg USE_SCCACHE=1 \
|
||||
--build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
|
||||
--build-arg SCCACHE_REGION_NAME=us-west-2 \
|
||||
--build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
|
||||
--build-arg INSTALL_LMCACHE=true \
|
||||
--tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rock \
|
||||
--target vllm-openai \
|
||||
--progress plain \
|
||||
-f docker/Dockerfile.rock .
|
||||
|
||||
docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rock
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
|
||||
- label: "Publish nightly ROCk image to DockerHub"
|
||||
depends_on:
|
||||
- build-rock-release-image
|
||||
if: build.env("NIGHTLY") == "1"
|
||||
agents:
|
||||
queue: small_cpu_queue_release
|
||||
commands:
|
||||
- "bash .buildkite/scripts/push-nightly-builds-rocm.sh rock rocm714"
|
||||
# Clean up old nightly builds (keep only last 14)
|
||||
- "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly-rocm714- vllm/vllm-openai-rocm"
|
||||
- "bash .buildkite/scripts/cleanup-nightly-builds.sh base-nightly-rocm714- vllm/vllm-openai-rocm"
|
||||
plugins:
|
||||
- docker-login#v3.0.0:
|
||||
username: vllmbot
|
||||
password-env: DOCKERHUB_TOKEN
|
||||
env:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
DOCKERHUB_USERNAME: "vllmbot"
|
||||
|
||||
# =============================================================================
|
||||
# Publish to DockerHub and PyPI (at the end so all builds complete first)
|
||||
# =============================================================================
|
||||
|
||||
@@ -2,61 +2,108 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
#
|
||||
# Push ROCm nightly base image and nightly image from ECR
|
||||
# to Docker Hub as vllm/vllm-openai-rocm:base-nightly and vllm/vllm-openai-rocm:nightly
|
||||
# and vllm/vllm-openai-rocm:base-nightly-<commit> and vllm/vllm-openai-rocm:nightly-<commit>.
|
||||
# Run when NIGHTLY=1 after build-rocm-release-image has pushed to ECR.
|
||||
# Push a ROCm-family nightly base image and nightly image from ECR to Docker Hub
|
||||
# under vllm/vllm-openai-rocm, as base-nightly[-<variant>], nightly[-<variant>]
|
||||
# and their -<commit> forms.
|
||||
# Run when NIGHTLY=1 after the matching build-*-release-image step has pushed to ECR.
|
||||
#
|
||||
# Usage: push-nightly-builds-rocm.sh [ECR_TAG_SUFFIX TAG_VARIANT]
|
||||
# Pass both arguments or neither. With none, the default ROCm apt build is
|
||||
# published as :nightly. The TheRock build passes "rock" (the source ECR tag
|
||||
# suffix) and "rocm714" (the Docker Hub tag flavor), giving :nightly-rocm714.
|
||||
#
|
||||
# Local testing (no push to Docker Hub):
|
||||
# BUILDKITE_COMMIT=<commit-with-rocm-image-in-ecr> DRY_RUN=1 bash .buildkite/scripts/push-nightly-builds-rocm.sh
|
||||
# BASE_ECR_IMAGE=<full-base-image-reference> \
|
||||
# BUILDKITE_COMMIT=<commit-with-image-in-ecr> DRY_RUN=1 \
|
||||
# bash .buildkite/scripts/push-nightly-builds-rocm.sh
|
||||
# Requires: AWS CLI configured (for ECR public login), Docker. For full run: Docker Hub login.
|
||||
|
||||
set -ex
|
||||
set -euxo pipefail
|
||||
|
||||
# Use BUILDKITE_COMMIT from env (required; set to a commit that has ROCm image in ECR for local test)
|
||||
BUILDKITE_COMMIT="${BUILDKITE_COMMIT:?Set BUILDKITE_COMMIT to the commit SHA that has the ROCm image in ECR (e.g. from a previous release pipeline run)}"
|
||||
usage() {
|
||||
echo "Usage: $0 [ECR_TAG_SUFFIX TAG_VARIANT]" >&2
|
||||
}
|
||||
|
||||
DOCKERHUB_REPO="vllm/vllm-openai-rocm"
|
||||
|
||||
case "$#" in
|
||||
0)
|
||||
ECR_TAG_SUFFIX="rocm"
|
||||
TAG_VARIANT=""
|
||||
;;
|
||||
2)
|
||||
if [[ -z "$1" || -z "$2" ]]; then
|
||||
usage
|
||||
exit 2
|
||||
fi
|
||||
ECR_TAG_SUFFIX="$1"
|
||||
TAG_VARIANT="$2"
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
# Use BUILDKITE_COMMIT from env (required; set to a commit that has the image in ECR for local test)
|
||||
BUILDKITE_COMMIT="${BUILDKITE_COMMIT:?Set BUILDKITE_COMMIT to the commit SHA that has the image in ECR (e.g. from a previous release pipeline run)}"
|
||||
DRY_RUN="${DRY_RUN:-0}"
|
||||
|
||||
# Get the base image ECR tag (set by build-rocm-release-image pipeline step)
|
||||
BASE_ORIG_TAG="$(buildkite-agent meta-data get rocm-base-ecr-tag 2>/dev/null || echo "")"
|
||||
if [ -z "$BASE_ORIG_TAG" ]; then
|
||||
echo "WARNING: rocm-base-ecr-tag metadata not found, falling back to commit-based tag"
|
||||
BASE_ORIG_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm-base"
|
||||
ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-release-repo"
|
||||
|
||||
# Get the base image ECR tag (set by the build-*-release-image pipeline step)
|
||||
BASE_METADATA_KEY="${ECR_TAG_SUFFIX}-base-ecr-tag"
|
||||
if [[ -n "${BASE_ECR_IMAGE:-}" ]]; then
|
||||
if [[ "$DRY_RUN" != "1" ]]; then
|
||||
echo "ERROR: BASE_ECR_IMAGE may only be used with DRY_RUN=1" >&2
|
||||
exit 1
|
||||
fi
|
||||
BASE_ORIG_TAG="$BASE_ECR_IMAGE"
|
||||
elif ! BASE_ORIG_TAG="$(buildkite-agent meta-data get "$BASE_METADATA_KEY")"; then
|
||||
echo "ERROR: Failed to read required Buildkite metadata '$BASE_METADATA_KEY'" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -z "$BASE_ORIG_TAG" ]]; then
|
||||
echo "ERROR: Required Buildkite metadata '$BASE_METADATA_KEY' is empty" >&2
|
||||
echo "Set BASE_ECR_IMAGE to the full ECR base image reference for local testing" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ORIG_TAG="${BUILDKITE_COMMIT}-rocm"
|
||||
BASE_TAG_NAME="base-nightly"
|
||||
TAG_NAME="nightly"
|
||||
BASE_TAG_NAME_COMMIT="base-nightly-${BUILDKITE_COMMIT}"
|
||||
TAG_NAME_COMMIT="nightly-${BUILDKITE_COMMIT}"
|
||||
ORIG_TAG="${BUILDKITE_COMMIT}-${ECR_TAG_SUFFIX}"
|
||||
VARIANT_SUFFIX=""
|
||||
[[ -n "$TAG_VARIANT" ]] && VARIANT_SUFFIX="-$TAG_VARIANT"
|
||||
BASE_TAG_NAME="base-nightly${VARIANT_SUFFIX}"
|
||||
TAG_NAME="nightly${VARIANT_SUFFIX}"
|
||||
BASE_TAG_NAME_COMMIT="base-nightly${VARIANT_SUFFIX}-${BUILDKITE_COMMIT}"
|
||||
TAG_NAME_COMMIT="nightly${VARIANT_SUFFIX}-${BUILDKITE_COMMIT}"
|
||||
|
||||
echo "Pushing ROCm base image from ECR: $BASE_ORIG_TAG"
|
||||
echo "Pushing ROCm release image from ECR tag: $ORIG_TAG to Docker Hub as $TAG_NAME and $TAG_NAME_COMMIT"
|
||||
echo "Pushing base image from ECR: $BASE_ORIG_TAG"
|
||||
echo "Pushing release image from ECR tag: $ORIG_TAG to Docker Hub as $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT"
|
||||
[[ "$DRY_RUN" == "1" ]] && echo "[DRY_RUN] Skipping push to Docker Hub"
|
||||
|
||||
# Login to ECR and pull the image built by build-rocm-release-image
|
||||
# Login to ECR and pull the images built by build-*-release-image
|
||||
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
|
||||
docker pull "$BASE_ORIG_TAG"
|
||||
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG"
|
||||
docker pull "${ECR_REPO}:${ORIG_TAG}"
|
||||
|
||||
# Tag for Docker Hub (base-nightly and base-nightly-<commit>, nightly and nightly-<commit>)
|
||||
docker tag "$BASE_ORIG_TAG" vllm/vllm-openai-rocm:"$BASE_TAG_NAME"
|
||||
docker tag "$BASE_ORIG_TAG" vllm/vllm-openai-rocm:"$BASE_TAG_NAME_COMMIT"
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG" vllm/vllm-openai-rocm:"$TAG_NAME"
|
||||
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG" vllm/vllm-openai-rocm:"$TAG_NAME_COMMIT"
|
||||
docker tag "$BASE_ORIG_TAG" "$DOCKERHUB_REPO":"$BASE_TAG_NAME"
|
||||
docker tag "$BASE_ORIG_TAG" "$DOCKERHUB_REPO":"$BASE_TAG_NAME_COMMIT"
|
||||
docker tag "${ECR_REPO}:${ORIG_TAG}" "$DOCKERHUB_REPO":"$TAG_NAME"
|
||||
docker tag "${ECR_REPO}:${ORIG_TAG}" "$DOCKERHUB_REPO":"$TAG_NAME_COMMIT"
|
||||
|
||||
if [[ "$DRY_RUN" == "1" ]]; then
|
||||
echo "[DRY_RUN] Would push vllm/vllm-openai-rocm:$BASE_TAG_NAME and vllm/vllm-openai-rocm:$BASE_TAG_NAME_COMMIT"
|
||||
echo "[DRY_RUN] Would push vllm/vllm-openai-rocm:$TAG_NAME and vllm/vllm-openai-rocm:$TAG_NAME_COMMIT"
|
||||
echo "[DRY_RUN] Would push $DOCKERHUB_REPO:$BASE_TAG_NAME and $DOCKERHUB_REPO:$BASE_TAG_NAME_COMMIT"
|
||||
echo "[DRY_RUN] Would push $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT"
|
||||
echo "[DRY_RUN] Local tags created. Exiting without push."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Push to Docker Hub (docker-login plugin runs before this step in CI)
|
||||
docker push vllm/vllm-openai-rocm:"$BASE_TAG_NAME"
|
||||
docker push vllm/vllm-openai-rocm:"$BASE_TAG_NAME_COMMIT"
|
||||
docker push vllm/vllm-openai-rocm:"$TAG_NAME"
|
||||
docker push vllm/vllm-openai-rocm:"$TAG_NAME_COMMIT"
|
||||
docker push "$DOCKERHUB_REPO":"$BASE_TAG_NAME"
|
||||
docker push "$DOCKERHUB_REPO":"$BASE_TAG_NAME_COMMIT"
|
||||
docker push "$DOCKERHUB_REPO":"$TAG_NAME"
|
||||
docker push "$DOCKERHUB_REPO":"$TAG_NAME_COMMIT"
|
||||
|
||||
echo "Pushed vllm/vllm-openai-rocm:$BASE_TAG_NAME and vllm/vllm-openai-rocm:$BASE_TAG_NAME_COMMIT"
|
||||
echo "Pushed vllm/vllm-openai-rocm:$TAG_NAME and vllm/vllm-openai-rocm:$TAG_NAME_COMMIT"
|
||||
echo "Pushed $DOCKERHUB_REPO:$BASE_TAG_NAME and $DOCKERHUB_REPO:$BASE_TAG_NAME_COMMIT"
|
||||
echo "Pushed $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT"
|
||||
|
||||
Reference in New Issue
Block a user