diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml index 17b6592ad5..24da6a14a1 100644 --- a/.buildkite/release-pipeline.yaml +++ b/.buildkite/release-pipeline.yaml @@ -959,6 +959,144 @@ steps: DOCKER_BUILDKIT: "1" DOCKERHUB_USERNAME: "vllmbot" + # ============================================================================= + # ROCk Release Pipeline (x86_64 only) + # ============================================================================= + # + # Same shape as the ROCm pipeline above (Jobs 1 and 6 only), but built from + # docker/Dockerfile.rock_base and docker/Dockerfile.rock, which install the + # ROCm SDK from TheRock wheels instead of the ROCm apt repos. + # + # Note: ROCm SDK version is determined by ROCM_SDK_VERSION in + # docker/Dockerfile.rock_base + # + # ============================================================================= + + - group: "Build ROCk Image" + key: "build-rock-image" + depends_on: ~ + steps: + # ROCk Job 1: Build ROCk Base Image (with ECR caching) + - label: ":rocm: Build ROCk Base Image" + id: build-rock-base-image + depends_on: ~ + agents: + queue: cpu_queue_release + timeout_in_minutes: 360 + commands: + - | + set -euo pipefail + + # The SDK version, torch wheels and component branches all live in + # the Dockerfile, so its hash is the cache key. + CACHE_KEY=$$(sha256sum docker/Dockerfile.rock_base | cut -c1-16) + ECR_CACHE_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:$${CACHE_KEY}-rock-base" + + echo "========================================" + echo "ROCk Base Build Configuration" + echo "========================================" + echo " CACHE_KEY: $${CACHE_KEY}" + echo " ECR_CACHE_TAG: $${ECR_CACHE_TAG}" + echo "========================================" + + # Login to ECR + aws ecr-public get-login-password --region us-east-1 | \ + docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7 + + if docker manifest inspect "$${ECR_CACHE_TAG}" > /dev/null 2>&1; then + echo "ECR image cache HIT" + else + echo "CACHE MISS - Building from scratch..." + + DOCKER_BUILDKIT=1 docker buildx build \ + --file docker/Dockerfile.rock_base \ + --tag "$${ECR_CACHE_TAG}" \ + --build-arg USE_SCCACHE=1 \ + --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \ + --build-arg SCCACHE_REGION_NAME=us-west-2 \ + --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \ + --push \ + . + fi + + # Save ECR tag for downstream jobs + buildkite-agent meta-data set "rock-base-image-tag" "$${ECR_CACHE_TAG}" + env: + DOCKER_BUILDKIT: "1" + + # ROCk Job 2: Build ROCk Release Docker Image + - label: ":docker: Build release image - x86_64 - ROCk" + id: build-rock-release-image + depends_on: + - step: block-build-release-images + allow_failure: true + - step: build-rock-base-image + allow_failure: false + agents: + queue: cpu_queue_release + timeout_in_minutes: 60 + commands: + - | + set -euo pipefail + + # Login to ECR + aws ecr-public get-login-password --region us-east-1 | \ + docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7 + + ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rock-base-image-tag 2>/dev/null || echo '')" + if [ -z "$${ECR_IMAGE_TAG}" ]; then + echo "ERROR: rock-base-image-tag metadata not found" + echo "This should have been set by the build-rock-base-image job" + exit 1 + fi + + docker pull "$${ECR_IMAGE_TAG}" + + # Pass the base image ECR tag to downstream steps (nightly publish) + buildkite-agent meta-data set "rock-base-ecr-tag" "$${ECR_IMAGE_TAG}" + + echo "========================================" + echo "Building vLLM ROCk release image with:" + echo " BASE_IMAGE: $${ECR_IMAGE_TAG}" + echo " BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}" + echo "========================================" + + DOCKER_BUILDKIT=1 docker build \ + --build-arg max_jobs=16 \ + --build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \ + --build-arg USE_SCCACHE=1 \ + --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \ + --build-arg SCCACHE_REGION_NAME=us-west-2 \ + --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \ + --build-arg INSTALL_LMCACHE=true \ + --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rock \ + --target vllm-openai \ + --progress plain \ + -f docker/Dockerfile.rock . + + docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rock + env: + DOCKER_BUILDKIT: "1" + + - label: "Publish nightly ROCk image to DockerHub" + depends_on: + - build-rock-release-image + if: build.env("NIGHTLY") == "1" + agents: + queue: small_cpu_queue_release + commands: + - "bash .buildkite/scripts/push-nightly-builds-rocm.sh rock rocm714" + # Clean up old nightly builds (keep only last 14) + - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly-rocm714- vllm/vllm-openai-rocm" + - "bash .buildkite/scripts/cleanup-nightly-builds.sh base-nightly-rocm714- vllm/vllm-openai-rocm" + plugins: + - docker-login#v3.0.0: + username: vllmbot + password-env: DOCKERHUB_TOKEN + env: + DOCKER_BUILDKIT: "1" + DOCKERHUB_USERNAME: "vllmbot" + # ============================================================================= # Publish to DockerHub and PyPI (at the end so all builds complete first) # ============================================================================= diff --git a/.buildkite/scripts/push-nightly-builds-rocm.sh b/.buildkite/scripts/push-nightly-builds-rocm.sh index 07577f8fd8..2a1a0b0804 100644 --- a/.buildkite/scripts/push-nightly-builds-rocm.sh +++ b/.buildkite/scripts/push-nightly-builds-rocm.sh @@ -2,61 +2,108 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # -# Push ROCm nightly base image and nightly image from ECR -# to Docker Hub as vllm/vllm-openai-rocm:base-nightly and vllm/vllm-openai-rocm:nightly -# and vllm/vllm-openai-rocm:base-nightly- and vllm/vllm-openai-rocm:nightly-. -# Run when NIGHTLY=1 after build-rocm-release-image has pushed to ECR. +# Push a ROCm-family nightly base image and nightly image from ECR to Docker Hub +# under vllm/vllm-openai-rocm, as base-nightly[-], nightly[-] +# and their - forms. +# Run when NIGHTLY=1 after the matching build-*-release-image step has pushed to ECR. +# +# Usage: push-nightly-builds-rocm.sh [ECR_TAG_SUFFIX TAG_VARIANT] +# Pass both arguments or neither. With none, the default ROCm apt build is +# published as :nightly. The TheRock build passes "rock" (the source ECR tag +# suffix) and "rocm714" (the Docker Hub tag flavor), giving :nightly-rocm714. # # Local testing (no push to Docker Hub): -# BUILDKITE_COMMIT= DRY_RUN=1 bash .buildkite/scripts/push-nightly-builds-rocm.sh +# BASE_ECR_IMAGE= \ +# BUILDKITE_COMMIT= DRY_RUN=1 \ +# bash .buildkite/scripts/push-nightly-builds-rocm.sh # Requires: AWS CLI configured (for ECR public login), Docker. For full run: Docker Hub login. -set -ex +set -euxo pipefail -# Use BUILDKITE_COMMIT from env (required; set to a commit that has ROCm image in ECR for local test) -BUILDKITE_COMMIT="${BUILDKITE_COMMIT:?Set BUILDKITE_COMMIT to the commit SHA that has the ROCm image in ECR (e.g. from a previous release pipeline run)}" +usage() { + echo "Usage: $0 [ECR_TAG_SUFFIX TAG_VARIANT]" >&2 +} + +DOCKERHUB_REPO="vllm/vllm-openai-rocm" + +case "$#" in + 0) + ECR_TAG_SUFFIX="rocm" + TAG_VARIANT="" + ;; + 2) + if [[ -z "$1" || -z "$2" ]]; then + usage + exit 2 + fi + ECR_TAG_SUFFIX="$1" + TAG_VARIANT="$2" + ;; + *) + usage + exit 2 + ;; +esac + +# Use BUILDKITE_COMMIT from env (required; set to a commit that has the image in ECR for local test) +BUILDKITE_COMMIT="${BUILDKITE_COMMIT:?Set BUILDKITE_COMMIT to the commit SHA that has the image in ECR (e.g. from a previous release pipeline run)}" DRY_RUN="${DRY_RUN:-0}" -# Get the base image ECR tag (set by build-rocm-release-image pipeline step) -BASE_ORIG_TAG="$(buildkite-agent meta-data get rocm-base-ecr-tag 2>/dev/null || echo "")" -if [ -z "$BASE_ORIG_TAG" ]; then - echo "WARNING: rocm-base-ecr-tag metadata not found, falling back to commit-based tag" - BASE_ORIG_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm-base" +ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-release-repo" + +# Get the base image ECR tag (set by the build-*-release-image pipeline step) +BASE_METADATA_KEY="${ECR_TAG_SUFFIX}-base-ecr-tag" +if [[ -n "${BASE_ECR_IMAGE:-}" ]]; then + if [[ "$DRY_RUN" != "1" ]]; then + echo "ERROR: BASE_ECR_IMAGE may only be used with DRY_RUN=1" >&2 + exit 1 + fi + BASE_ORIG_TAG="$BASE_ECR_IMAGE" +elif ! BASE_ORIG_TAG="$(buildkite-agent meta-data get "$BASE_METADATA_KEY")"; then + echo "ERROR: Failed to read required Buildkite metadata '$BASE_METADATA_KEY'" >&2 + exit 1 +fi +if [[ -z "$BASE_ORIG_TAG" ]]; then + echo "ERROR: Required Buildkite metadata '$BASE_METADATA_KEY' is empty" >&2 + echo "Set BASE_ECR_IMAGE to the full ECR base image reference for local testing" >&2 + exit 1 fi -ORIG_TAG="${BUILDKITE_COMMIT}-rocm" -BASE_TAG_NAME="base-nightly" -TAG_NAME="nightly" -BASE_TAG_NAME_COMMIT="base-nightly-${BUILDKITE_COMMIT}" -TAG_NAME_COMMIT="nightly-${BUILDKITE_COMMIT}" +ORIG_TAG="${BUILDKITE_COMMIT}-${ECR_TAG_SUFFIX}" +VARIANT_SUFFIX="" +[[ -n "$TAG_VARIANT" ]] && VARIANT_SUFFIX="-$TAG_VARIANT" +BASE_TAG_NAME="base-nightly${VARIANT_SUFFIX}" +TAG_NAME="nightly${VARIANT_SUFFIX}" +BASE_TAG_NAME_COMMIT="base-nightly${VARIANT_SUFFIX}-${BUILDKITE_COMMIT}" +TAG_NAME_COMMIT="nightly${VARIANT_SUFFIX}-${BUILDKITE_COMMIT}" -echo "Pushing ROCm base image from ECR: $BASE_ORIG_TAG" -echo "Pushing ROCm release image from ECR tag: $ORIG_TAG to Docker Hub as $TAG_NAME and $TAG_NAME_COMMIT" +echo "Pushing base image from ECR: $BASE_ORIG_TAG" +echo "Pushing release image from ECR tag: $ORIG_TAG to Docker Hub as $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT" [[ "$DRY_RUN" == "1" ]] && echo "[DRY_RUN] Skipping push to Docker Hub" -# Login to ECR and pull the image built by build-rocm-release-image +# Login to ECR and pull the images built by build-*-release-image aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7 docker pull "$BASE_ORIG_TAG" -docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG" +docker pull "${ECR_REPO}:${ORIG_TAG}" # Tag for Docker Hub (base-nightly and base-nightly-, nightly and nightly-) -docker tag "$BASE_ORIG_TAG" vllm/vllm-openai-rocm:"$BASE_TAG_NAME" -docker tag "$BASE_ORIG_TAG" vllm/vllm-openai-rocm:"$BASE_TAG_NAME_COMMIT" -docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG" vllm/vllm-openai-rocm:"$TAG_NAME" -docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:"$ORIG_TAG" vllm/vllm-openai-rocm:"$TAG_NAME_COMMIT" +docker tag "$BASE_ORIG_TAG" "$DOCKERHUB_REPO":"$BASE_TAG_NAME" +docker tag "$BASE_ORIG_TAG" "$DOCKERHUB_REPO":"$BASE_TAG_NAME_COMMIT" +docker tag "${ECR_REPO}:${ORIG_TAG}" "$DOCKERHUB_REPO":"$TAG_NAME" +docker tag "${ECR_REPO}:${ORIG_TAG}" "$DOCKERHUB_REPO":"$TAG_NAME_COMMIT" if [[ "$DRY_RUN" == "1" ]]; then - echo "[DRY_RUN] Would push vllm/vllm-openai-rocm:$BASE_TAG_NAME and vllm/vllm-openai-rocm:$BASE_TAG_NAME_COMMIT" - echo "[DRY_RUN] Would push vllm/vllm-openai-rocm:$TAG_NAME and vllm/vllm-openai-rocm:$TAG_NAME_COMMIT" + echo "[DRY_RUN] Would push $DOCKERHUB_REPO:$BASE_TAG_NAME and $DOCKERHUB_REPO:$BASE_TAG_NAME_COMMIT" + echo "[DRY_RUN] Would push $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT" echo "[DRY_RUN] Local tags created. Exiting without push." exit 0 fi # Push to Docker Hub (docker-login plugin runs before this step in CI) -docker push vllm/vllm-openai-rocm:"$BASE_TAG_NAME" -docker push vllm/vllm-openai-rocm:"$BASE_TAG_NAME_COMMIT" -docker push vllm/vllm-openai-rocm:"$TAG_NAME" -docker push vllm/vllm-openai-rocm:"$TAG_NAME_COMMIT" +docker push "$DOCKERHUB_REPO":"$BASE_TAG_NAME" +docker push "$DOCKERHUB_REPO":"$BASE_TAG_NAME_COMMIT" +docker push "$DOCKERHUB_REPO":"$TAG_NAME" +docker push "$DOCKERHUB_REPO":"$TAG_NAME_COMMIT" -echo "Pushed vllm/vllm-openai-rocm:$BASE_TAG_NAME and vllm/vllm-openai-rocm:$BASE_TAG_NAME_COMMIT" -echo "Pushed vllm/vllm-openai-rocm:$TAG_NAME and vllm/vllm-openai-rocm:$TAG_NAME_COMMIT" +echo "Pushed $DOCKERHUB_REPO:$BASE_TAG_NAME and $DOCKERHUB_REPO:$BASE_TAG_NAME_COMMIT" +echo "Pushed $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT"