diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml index 4bf7e3ae7d..4af4eaf56e 100644 --- a/.buildkite/release-pipeline.yaml +++ b/.buildkite/release-pipeline.yaml @@ -439,43 +439,48 @@ steps: env: DOCKER_BUILDKIT: "1" + # CPU release images follow the CUDA pattern: a single build label per arch + # pushes the arch-specific commit tag to ECR, consumed by both the nightly + # DockerHub publish and the release publish step. The block auto-skips in + # nightly (if NIGHTLY != "1") so the build runs unattended, mirroring + # block-build-release-images. - block: "Build release image for x86_64 CPU" key: block-cpu-release-image-build depends_on: ~ + if: build.env("NIGHTLY") != "1" - label: "Build release image - x86_64 - CPU" key: build-cpu-release-image-x86 depends_on: - block-cpu-release-image-build - - input-release-version + allow_dependency_failure: true agents: queue: cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_CPU_X86=true --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version) --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:latest --progress plain --target vllm-openai -f docker/Dockerfile.cpu ." - - "docker push public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:latest" - - "docker push public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version)" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$(buildkite-agent meta-data get release-version)" release-images' + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --build-arg VLLM_CPU_X86=true --tag public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$BUILDKITE_COMMIT-x86_64 --progress plain --target vllm-openai -f docker/Dockerfile.cpu ." + - "docker push public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$BUILDKITE_COMMIT-x86_64" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:$BUILDKITE_COMMIT-x86_64" release-images' env: DOCKER_BUILDKIT: "1" - block: "Build release image for arm64 CPU" key: block-arm64-cpu-release-image-build depends_on: ~ + if: build.env("NIGHTLY") != "1" - label: "Build release image - arm64 - CPU" key: build-cpu-release-image-arm64 depends_on: - block-arm64-cpu-release-image-build - - input-release-version + allow_dependency_failure: true agents: queue: arm64_cpu_queue_release commands: - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7" - - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --tag public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$(buildkite-agent meta-data get release-version) --tag public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:latest --progress plain --target vllm-openai -f docker/Dockerfile.cpu ." - - "docker push public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:latest" - - "docker push public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$(buildkite-agent meta-data get release-version)" - - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$(buildkite-agent meta-data get release-version)" release-images' + - "DOCKER_BUILDKIT=1 docker build --build-arg max_jobs=16 --build-arg GIT_REPO_CHECK=1 --tag public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$BUILDKITE_COMMIT-arm64 --progress plain --target vllm-openai -f docker/Dockerfile.cpu ." + - "docker push public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$BUILDKITE_COMMIT-arm64" + - 'bash .buildkite/scripts/annotate-build-artifact.sh "$$BUILDKITE_LABEL" "public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:$BUILDKITE_COMMIT-arm64" release-images' env: DOCKER_BUILDKIT: "1" @@ -580,6 +585,25 @@ steps: DOCKER_BUILDKIT: "1" DOCKERHUB_USERNAME: "vllmbot" + - label: "Publish nightly CPU multi-arch image to DockerHub" + depends_on: + - build-cpu-release-image-x86 + - build-cpu-release-image-arm64 + if: build.env("NIGHTLY") == "1" + agents: + queue: small_cpu_queue_release + commands: + - "bash .buildkite/scripts/push-nightly-builds-cpu.sh" + # Clean up old nightly builds (keep only last 14) + - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly- vllm/vllm-openai-cpu" + plugins: + - docker-login#v3.0.0: + username: vllmbot + password-env: DOCKERHUB_TOKEN + env: + DOCKER_BUILDKIT: "1" + DOCKERHUB_USERNAME: "vllmbot" + # ============================================================================= # ROCm Release Pipeline (x86_64 only) # ============================================================================= diff --git a/.buildkite/scripts/publish-release-images.sh b/.buildkite/scripts/publish-release-images.sh index 514ee99b38..2fa334241f 100755 --- a/.buildkite/scripts/publish-release-images.sh +++ b/.buildkite/scripts/publish-release-images.sh @@ -181,8 +181,8 @@ fi # `:latest` multi-arch manifest still resolves to the previous release. if target_enabled cpu; then - CPU_X86_TAG=public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v${RELEASE_VERSION} - CPU_ARM_TAG=public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:v${RELEASE_VERSION} + CPU_X86_TAG=public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:${COMMIT}-x86_64 + CPU_ARM_TAG=public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:${COMMIT}-arm64 CPU_X86_AVAILABLE=false CPU_ARM_AVAILABLE=false diff --git a/.buildkite/scripts/push-nightly-builds-cpu.sh b/.buildkite/scripts/push-nightly-builds-cpu.sh new file mode 100755 index 0000000000..e9c346df56 --- /dev/null +++ b/.buildkite/scripts/push-nightly-builds-cpu.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# +# Push CPU nightly images from ECR to Docker Hub as +# vllm/vllm-openai-cpu:nightly and vllm/vllm-openai-cpu:nightly-, +# combining the x86_64 and aarch64 architectures into a multi-arch manifest. +# Run when NIGHTLY=1 after the CPU release images have been pushed to ECR. +# +# Local testing (no push to Docker Hub): +# BUILDKITE_COMMIT= DRY_RUN=1 bash .buildkite/scripts/push-nightly-builds-cpu.sh +# Requires: AWS CLI configured (for ECR public login), Docker. For full run: Docker Hub login. + +set -ex + +# Use BUILDKITE_COMMIT from env (required; set to a commit that has CPU images in ECR for local test) +BUILDKITE_COMMIT="${BUILDKITE_COMMIT:?Set BUILDKITE_COMMIT to the commit SHA that has the CPU images in ECR (e.g. from a previous release pipeline run)}" +DRY_RUN="${DRY_RUN:-0}" + +TAG_NAME="nightly" +TAG_NAME_COMMIT="nightly-${BUILDKITE_COMMIT}" + +# arch-dependent source images in ECR (pushed by the CPU release image build steps) +X86_ORIG_TAG="public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:${BUILDKITE_COMMIT}-x86_64" +ARM64_ORIG_TAG="public.ecr.aws/q9t5s3a7/vllm-arm64-cpu-release-repo:${BUILDKITE_COMMIT}-arm64" + +echo "Pushing CPU release images from ECR to Docker Hub as $TAG_NAME and $TAG_NAME_COMMIT" +echo " x86_64: $X86_ORIG_TAG" +echo " arm64: $ARM64_ORIG_TAG" +[[ "$DRY_RUN" == "1" ]] && echo "[DRY_RUN] Skipping push to Docker Hub" + +# Login to ECR and pull the arch-dependent images +aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7 +docker pull "$X86_ORIG_TAG" +docker pull "$ARM64_ORIG_TAG" + +# Tag arch-dependent images for Docker Hub +docker tag "$X86_ORIG_TAG" vllm/vllm-openai-cpu:"$TAG_NAME"-x86_64 +docker tag "$ARM64_ORIG_TAG" vllm/vllm-openai-cpu:"$TAG_NAME"-arm64 + +if [[ "$DRY_RUN" == "1" ]]; then + echo "[DRY_RUN] Would push vllm/vllm-openai-cpu:$TAG_NAME-x86_64 and vllm/vllm-openai-cpu:$TAG_NAME-arm64" + echo "[DRY_RUN] Would create and push manifests vllm/vllm-openai-cpu:$TAG_NAME and vllm/vllm-openai-cpu:$TAG_NAME_COMMIT" + echo "[DRY_RUN] Local tags created. Exiting without push." + exit 0 +fi + +# Push arch-dependent images to Docker Hub +docker push vllm/vllm-openai-cpu:"$TAG_NAME"-x86_64 +docker push vllm/vllm-openai-cpu:"$TAG_NAME"-arm64 + +# Create and push the arch-independent manifests +docker manifest create vllm/vllm-openai-cpu:"$TAG_NAME" vllm/vllm-openai-cpu:"$TAG_NAME"-x86_64 vllm/vllm-openai-cpu:"$TAG_NAME"-arm64 --amend +docker manifest create vllm/vllm-openai-cpu:"$TAG_NAME_COMMIT" vllm/vllm-openai-cpu:"$TAG_NAME"-x86_64 vllm/vllm-openai-cpu:"$TAG_NAME"-arm64 --amend +docker manifest push vllm/vllm-openai-cpu:"$TAG_NAME" +docker manifest push vllm/vllm-openai-cpu:"$TAG_NAME_COMMIT" + +echo "Pushed vllm/vllm-openai-cpu:$TAG_NAME and vllm/vllm-openai-cpu:$TAG_NAME_COMMIT"