iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[ROCm][CI] Make ROCm 10.0 the default image; keep ROCm 7.2 as -rocm72 (#58761)

Browse Source 
Signed-off-by: Rohan Potdar <rohan.potdar@amd.com>
Signed-off-by: Rohan138 <rohanpotdar138@gmail.com>
Signed-off-by: Micah Williamson <micah.williamson@amd.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Micah Williamson <micah.williamson@amd.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Rohan Potdar
2026-10-17 21:43:59 -05:00
committed by GitHub
co-authored by Claude Opus 5.5 Micah Williamson Kevin H. Luu Andreas Karatzas
parent 03ee2e134d
commit b2456c622b
24 changed files with 1742 additions and 1634 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.buildkite/release-pipeline.yaml
+208 -473
View File
@@ -703,385 +703,6 @@ steps:
DOCKER_BUILDKIT: "1"
DOCKERHUB_USERNAME: "vllmbot"
# =============================================================================
# ROCm Release Pipeline (x86_64 only)
# =============================================================================
#
# vLLM version is determined by the Buildkite checkout (like CUDA pipeline).
# To build a specific version, trigger the build from that branch/tag.
#
# Environment variables for ROCm builds (set via Buildkite UI or schedule):
#
# Note: ROCm version is determined by BASE_IMAGE in docker/Dockerfile.rocm_base
#
# =============================================================================
- group: "Build ROCm Wheel / Image "
key: "build-rocm-wheel-image"
depends_on: ~
steps:
# ROCm Job 1: Build ROCm Base Wheels (with S3 caching)
- label: ":rocm: Build ROCm Base Image & Wheels"
id: build-rocm-base-wheels
depends_on: ~
agents:
queue: cpu_queue_release
commands:
- |
set -euo pipefail
# Generate cache key
CACHE_KEY=$$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
ECR_CACHE_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:$${CACHE_KEY}-rocm-base"
echo "========================================"
echo "ROCm Base Build Configuration"
echo "========================================"
echo " CACHE_KEY: $${CACHE_KEY}"
echo " ECR_CACHE_TAG: $${ECR_CACHE_TAG}"
echo "========================================"
# Login to ECR
aws ecr-public get-login-password --region us-east-1 | \
docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
IMAGE_EXISTS=false
WHEELS_EXIST=false
# Check ECR for Docker image
if docker manifest inspect "$${ECR_CACHE_TAG}" > /dev/null 2>&1; then
IMAGE_EXISTS=true
echo "ECR image cache HIT"
fi
# Check S3 for wheels
WHEEL_CACHE_STATUS=$(.buildkite/scripts/cache-rocm-base-wheels.sh check)
if [ "$${WHEEL_CACHE_STATUS}" = "hit" ]; then
WHEELS_EXIST=true
echo "S3 wheels cache HIT"
fi
# Scenario 1: Both cached (best case)
if [ "$${IMAGE_EXISTS}" = "true" ] && [ "$${WHEELS_EXIST}" = "true" ]; then
echo ""
echo "FULL CACHE HIT - Reusing both image and wheels"
echo ""
# Download wheels
.buildkite/scripts/cache-rocm-base-wheels.sh download
# Save ECR tag for downstream jobs
buildkite-agent meta-data set "rocm-base-image-tag" "$${ECR_CACHE_TAG}"
# Scenario 2: Full rebuild needed
else
echo ""
echo " CACHE MISS - Building from scratch..."
echo ""
# Build full base image and push to ECR
DOCKER_BUILDKIT=1 docker buildx build \
--file docker/Dockerfile.rocm_base \
--tag "$${ECR_CACHE_TAG}" \
--build-arg USE_SCCACHE=1 \
--build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
--build-arg SCCACHE_REGION_NAME=us-west-2 \
--build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
--push \
.
# Build wheel extraction stage
DOCKER_BUILDKIT=1 docker buildx build \
--file docker/Dockerfile.rocm_base \
--tag rocm-base-debs:$${BUILDKITE_BUILD_NUMBER} \
--target debs_wheel_release \
--build-arg USE_SCCACHE=1 \
--build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
--build-arg SCCACHE_REGION_NAME=us-west-2 \
--build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
--load \
.
# Extract and upload wheels
mkdir -p artifacts/rocm-base-wheels
cid=$(docker create rocm-base-debs:$${BUILDKITE_BUILD_NUMBER})
docker cp $${cid}:/app/debs/. artifacts/rocm-base-wheels/
docker rm $${cid}
.buildkite/scripts/cache-rocm-base-wheels.sh upload
# Cache base docker image to ECR
docker push "$${ECR_CACHE_TAG}"
buildkite-agent meta-data set "rocm-base-image-tag" "$${ECR_CACHE_TAG}"
echo ""
echo " Build complete - Image and wheels cached"
fi
artifact_paths:
- "artifacts/rocm-base-wheels/*.whl"
env:
DOCKER_BUILDKIT: "1"
S3_BUCKET: "vllm-wheels"
# ROCm Job 2: Build vLLM ROCm Wheel
- label: ":python: Build vLLM ROCm Wheel - x86_64"
id: build-rocm-vllm-wheel
depends_on:
- step: build-rocm-base-wheels
allow_failure: false
agents:
queue: cpu_queue_release
timeout_in_minutes: 180
commands:
# Download artifacts and prepare Docker image
- |
set -euo pipefail
# Ensure git tags are up-to-date (Buildkite's default fetch doesn't update tags)
# This fixes version detection when tags are moved/force-pushed
echo "Fetching latest tags from origin..."
git fetch --tags --force origin
# Log tag information for debugging version detection
echo "========================================"
echo "Git Tag Verification"
echo "========================================"
echo "Current HEAD: $(git rev-parse HEAD)"
echo "git describe --tags: $(git describe --tags 2>/dev/null || echo 'No tags found')"
echo ""
echo "Recent tags (pointing to commits near HEAD):"
git tag -l --sort=-creatordate | head -5
echo "setuptools_scm version detection:"
pip install -q setuptools_scm 2>/dev/null || true
python3 -c "import setuptools_scm; print(' Detected version:', setuptools_scm.get_version())" 2>/dev/null || echo " (setuptools_scm not available in this environment)"
echo "========================================"
# Download wheel artifacts from current build
echo "Downloading wheel artifacts from current build"
buildkite-agent artifact download "artifacts/rocm-base-wheels/*.whl" .
# Get ECR image tag from metadata (set by build-rocm-base-wheels)
ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rocm-base-image-tag 2>/dev/null || echo '')"
if [ -z "$${ECR_IMAGE_TAG}" ]; then
echo "ERROR: rocm-base-image-tag metadata not found"
echo "This should have been set by the build-rocm-base-wheels job"
exit 1
fi
echo "Pulling base Docker image from ECR: $${ECR_IMAGE_TAG}"
# Login to ECR
aws ecr-public get-login-password --region us-east-1 | \
docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
# Pull base Docker image from ECR
docker pull "$${ECR_IMAGE_TAG}"
echo "Loaded base image: $${ECR_IMAGE_TAG}"
# Prepare base wheels for Docker build context
mkdir -p docker/context/base-wheels
touch docker/context/base-wheels/.keep
cp artifacts/rocm-base-wheels/*.whl docker/context/base-wheels/
echo "Base wheels for vLLM build:"
ls -lh docker/context/base-wheels/
echo "========================================"
echo "Building vLLM wheel with:"
echo " BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}"
echo " BUILDKITE_BRANCH: $${BUILDKITE_BRANCH}"
echo " BASE_IMAGE: $${ECR_IMAGE_TAG}"
echo "========================================"
# Build vLLM wheel using local checkout (REMOTE_VLLM=0)
DOCKER_BUILDKIT=1 docker build \
--file docker/Dockerfile.rocm \
--target export_vllm_wheel_release \
--output type=local,dest=rocm-dist \
--build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \
--build-arg REMOTE_VLLM=0 \
--build-arg GIT_REPO_CHECK=1 \
--build-arg USE_SCCACHE=1 \
--build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
--build-arg SCCACHE_REGION_NAME=us-west-2 \
--build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
.
echo "Built vLLM wheel:"
ls -lh rocm-dist/*.whl
# Copy wheel to artifacts directory
mkdir -p artifacts/rocm-vllm-wheel
cp rocm-dist/*.whl artifacts/rocm-vllm-wheel/
echo "Final vLLM wheel:"
ls -lh artifacts/rocm-vllm-wheel/
artifact_paths:
- "artifacts/rocm-vllm-wheel/*.whl"
env:
DOCKER_BUILDKIT: "1"
S3_BUCKET: "vllm-wheels"
# ROCm Job 3: Upload Wheels to S3
- label: ":s3: Upload ROCm Wheels to S3"
id: upload-rocm-wheels
depends_on:
- step: build-rocm-vllm-wheel
allow_failure: false
agents:
queue: cpu_queue_release
timeout_in_minutes: 60
commands:
# Download all wheel artifacts and run upload
- |
set -euo pipefail
# Download artifacts from current build
echo "Downloading artifacts from current build"
buildkite-agent artifact download "artifacts/rocm-base-wheels/*.whl" .
buildkite-agent artifact download "artifacts/rocm-vllm-wheel/*.whl" .
# # Run upload script
bash .buildkite/scripts/upload-rocm-wheels.sh
env:
DOCKER_BUILDKIT: "1"
S3_BUCKET: "vllm-wheels"
# ROCm Job 4: Annotate ROCm Wheel Release
- label: ":memo: Annotate ROCm wheel release"
id: annotate-rocm-release
depends_on:
- upload-rocm-wheels
agents:
queue: cpu_queue_release
commands:
- "bash .buildkite/scripts/annotate-rocm-release.sh"
env:
S3_BUCKET: "vllm-wheels"
# ROCm Job 5: Generate Root Index for ROCm Wheels (for release only)
# This is the job to create https://wheels.vllm.ai/rocm/ index allowing
# users to install with `uv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm/`
- block: "Generate Root Index for ROCm Wheels for Release"
key: block-generate-root-index-rocm-wheels
depends_on: upload-rocm-wheels
- label: ":package: Generate Root Index for ROCm Wheels for Release"
depends_on: block-generate-root-index-rocm-wheels
id: generate-root-index-rocm-wheels
agents:
queue: cpu_queue_release
commands:
- "bash tools/vllm-rocm/generate-rocm-wheels-root-index.sh"
env:
S3_BUCKET: "vllm-wheels"
VARIANT: "rocm723"
# ROCm Job 6: Build ROCm Release Docker Image
- label: ":docker: Build release image - x86_64 - ROCm"
id: build-rocm-release-image
depends_on:
- step: block-build-release-images
allow_failure: true
- step: build-rocm-base-wheels
allow_failure: false
agents:
queue: cpu_queue_release
timeout_in_minutes: 60
commands:
- |
set -euo pipefail
# Login to ECR
aws ecr-public get-login-password --region us-east-1 | \
docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
# Get ECR image tag from metadata (set by build-rocm-base-wheels)
ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rocm-base-image-tag 2>/dev/null || echo '')"
if [ -z "$${ECR_IMAGE_TAG}" ]; then
echo "ERROR: rocm-base-image-tag metadata not found"
echo "This should have been set by the build-rocm-base-wheels job"
exit 1
fi
echo "Pulling base Docker image from ECR: $${ECR_IMAGE_TAG}"
# Pull base Docker image from ECR
docker pull "$${ECR_IMAGE_TAG}"
echo "Loaded base image: $${ECR_IMAGE_TAG}"
# Pass the base image ECR tag to downstream steps (nightly publish)
buildkite-agent meta-data set "rocm-base-ecr-tag" "$${ECR_IMAGE_TAG}"
echo "========================================"
echo "Building vLLM ROCm release image with:"
echo " BASE_IMAGE: $${ECR_IMAGE_TAG}"
echo " BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}"
echo "========================================"
# Build vLLM ROCm release image using cached base
DOCKER_BUILDKIT=1 docker build \
--build-arg max_jobs=16 \
--build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \
--build-arg USE_SCCACHE=1 \
--build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
--build-arg SCCACHE_REGION_NAME=us-west-2 \
--build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
--build-arg INSTALL_LMCACHE=true \
--tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm \
--target vllm-openai \
--progress plain \
-f docker/Dockerfile.rocm .
# Push to ECR
docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm
echo ""
echo " Successfully built and pushed ROCm release image"
echo " Image: public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm"
echo ""
env:
DOCKER_BUILDKIT: "1"
S3_BUCKET: "vllm-wheels"
- label: "Publish nightly XPU image to DockerHub"
depends_on:
- create-manifest-xpu
if: build.env("NIGHTLY") == "1"
agents:
queue: small_cpu_queue_release
commands:
- "bash .buildkite/scripts/xpu/push-nightly-builds-xpu.sh"
- "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly- vllm/vllm-openai-xpu"
plugins:
- docker-login#v3.0.0:
username: vllmbot
password-env: DOCKERHUB_TOKEN
env:
DOCKER_BUILDKIT: "1"
DOCKERHUB_USERNAME: "vllmbot"
- label: "Publish nightly ROCm image to DockerHub"
depends_on:
- build-rocm-release-image
if: build.env("NIGHTLY") == "1"
agents:
queue: small_cpu_queue_release
commands:
- "bash .buildkite/scripts/push-nightly-builds-rocm.sh"
# Clean up old nightly builds (keep only last 14)
- "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly- vllm/vllm-openai-rocm"
- "bash .buildkite/scripts/cleanup-nightly-builds.sh base-nightly- vllm/vllm-openai-rocm"
plugins:
- docker-login#v3.0.0:
username: vllmbot
password-env: DOCKERHUB_TOKEN
env:
DOCKER_BUILDKIT: "1"
DOCKERHUB_USERNAME: "vllmbot"
- trigger: "perf-eval"
label: "Trigger perf-eval (CUDA)"
key: trigger-perf-eval-cuda
@@ -1103,7 +724,7 @@ steps:
key: trigger-perf-eval-rocm
if: build.env("NIGHTLY") == "1" || build.env("RUN_PERF_EVAL") == "1"
depends_on:
- build-rocm-release-image
- build-rocm100-release-image
async: true
soft_fail: true
build:
@@ -1112,138 +733,250 @@ steps:
env:
NIGHTLY: 1 # run all perf-eval workloads
VLLM_COMMIT: "$BUILDKITE_COMMIT"
VLLM_IMAGE_ROCM: "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-rocm"
VLLM_IMAGE_ROCM: "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-rocm100"
# =============================================================================
# ROCk Release Pipeline (x86_64 only)
# ROCm Release Pipelines (x86_64 only)
# =============================================================================
#
# Same shape as the ROCm pipeline above (Jobs 1 and 6 only), but built from
# docker/Dockerfile.rock_base and docker/Dockerfile.rock, which install the
# ROCm SDK from TheRock wheels instead of the ROCm apt repos.
# One group per ROCm stack, identified by its base Dockerfile (ROCM_BASE_DOCKERFILE).
# The scripts derive the ROCm version, wheel variant, cache keys and image tags from
# it (.buildkite/scripts/rocm/stack.sh), so both groups share one set of step
# definitions, anchored on the ROCm 10.0 group.
#
# Note: ROCm SDK version is determined by ROCM_SDK_VERSION in
# docker/Dockerfile.rock_base
# - ROCm 10.0 (TheRock, docker/Dockerfile.rocm{,_base}) is the default: it owns the
# :nightly / :latest images and the root wheel index.
# - ROCm 7.2 (docker/Dockerfile.rocm_72{,_base}) is kept for a transition period as
# rocm72 images and wheels.
#
# vLLM version is determined by the Buildkite checkout (like the CUDA pipeline).
# =============================================================================
- group: "Build ROCk Image"
key: "build-rock-image"
- group: "Build ROCm 10.0 Wheel / Image"
key: "build-rocm100"
depends_on: ~
steps:
# ROCk Job 1: Build ROCk Base Image (with ECR caching)
- label: ":rocm: Build ROCk Base Image"
id: build-rock-base-image
- label: ":rocm: Build ROCm 10.0 Base Image & Wheels"
id: build-rocm100-base
depends_on: ~
agents:
agents: &rocm_build_agents
queue: cpu_queue_release
timeout_in_minutes: 360
commands:
env: &rocm100_env
ROCM_BASE_DOCKERFILE: "docker/Dockerfile.rocm_base"
ROCM_DEFAULT_ARG: "--default"
DOCKER_BUILDKIT: "1"
S3_BUCKET: "vllm-wheels"
DOCKERHUB_USERNAME: "vllmbot"
commands: &rocm_base_commands
- |
set -euo pipefail
# The SDK version, torch wheels and component branches all live in
# the Dockerfile, so its hash is the cache key.
CACHE_KEY=$$(sha256sum docker/Dockerfile.rock_base | cut -c1-16)
ECR_CACHE_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:$${CACHE_KEY}-rock-base"
echo "========================================"
echo "ROCk Base Build Configuration"
echo "========================================"
echo " CACHE_KEY: $${CACHE_KEY}"
echo " ECR_CACHE_TAG: $${ECR_CACHE_TAG}"
echo "========================================"
# Login to ECR
source .buildkite/scripts/rocm/stack.sh "$${ROCM_BASE_DOCKERFILE}"
SCCACHE_ARGS=(--build-arg USE_SCCACHE=1 --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache
--build-arg SCCACHE_REGION_NAME=us-west-2 --build-arg SCCACHE_S3_NO_CREDENTIALS=0)
aws ecr-public get-login-password --region us-east-1 | \
docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
if docker manifest inspect "$${ECR_CACHE_TAG}" > /dev/null 2>&1; then
echo "ECR image cache HIT"
# Base image, cached in ECR by the base Dockerfile's hash
if docker manifest inspect "$${ROCM_STACK_ECR_BASE}" > /dev/null 2>&1; then
echo "ECR image cache HIT: $${ROCM_STACK_ECR_BASE}"
else
echo "CACHE MISS - Building from scratch..."
DOCKER_BUILDKIT=1 docker buildx build \
--file docker/Dockerfile.rock_base \
--tag "$${ECR_CACHE_TAG}" \
--build-arg USE_SCCACHE=1 \
--build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
--build-arg SCCACHE_REGION_NAME=us-west-2 \
--build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
--push \
.
docker buildx build --file "$${ROCM_BASE_DOCKERFILE}" --tag "$${ROCM_STACK_ECR_BASE}" \
"$${SCCACHE_ARGS[@]}" --push .
fi
# Save ECR tag for downstream jobs
buildkite-agent meta-data set "rock-base-image-tag" "$${ECR_CACHE_TAG}"
env:
DOCKER_BUILDKIT: "1"
# Base wheels, cached in S3 (see cache-rocm-base-wheels.sh)
CACHE=.buildkite/scripts/cache-rocm-base-wheels.sh
if [ "$$($${CACHE} "$${ROCM_BASE_DOCKERFILE}" check)" = "hit" ]; then
$${CACHE} "$${ROCM_BASE_DOCKERFILE}" download
else
DEBS="rocm-base-debs:$${BUILDKITE_BUILD_NUMBER}-$${ROCM_STACK_VARIANT}"
docker buildx build --file "$${ROCM_BASE_DOCKERFILE}" --tag "$${DEBS}" \
--target debs_wheel_release "$${SCCACHE_ARGS[@]}" --load .
mkdir -p "$${ROCM_STACK_BASE_WHEELS_DIR}"
cid=$$(docker create "$${DEBS}")
docker cp "$${cid}":/app/debs/. "$${ROCM_STACK_BASE_WHEELS_DIR}/"
docker rm "$${cid}"
$${CACHE} "$${ROCM_BASE_DOCKERFILE}" upload
fi
buildkite-agent artifact upload "$${ROCM_STACK_BASE_WHEELS_DIR}/*.whl"
# ROCk Job 2: Build ROCk Release Docker Image
- label: ":docker: Build release image - x86_64 - ROCk"
id: build-rock-release-image
- label: ":python: Build vLLM ROCm 10.0 Wheel - x86_64"
id: build-rocm100-vllm-wheel
depends_on: build-rocm100-base
agents: *rocm_build_agents
timeout_in_minutes: 180
env: *rocm100_env
commands: &rocm_vllm_wheel_commands
- |
set -euo pipefail
source .buildkite/scripts/rocm/stack.sh "$${ROCM_BASE_DOCKERFILE}"
# Buildkite's default fetch doesn't update moved tags; version detection needs them
git fetch --tags --force origin
buildkite-agent artifact download "$${ROCM_STACK_BASE_WHEELS_DIR}/*.whl" .
aws ecr-public get-login-password --region us-east-1 | \
docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
docker pull "$${ROCM_STACK_ECR_BASE}"
mkdir -p docker/context/base-wheels
touch docker/context/base-wheels/.keep
cp "$${ROCM_STACK_BASE_WHEELS_DIR}"/*.whl docker/context/base-wheels/
docker build --file "$${ROCM_STACK_DOCKERFILE}" --target export_vllm_wheel_release \
--output type=local,dest=rocm-dist \
--build-arg BASE_IMAGE="$${ROCM_STACK_ECR_BASE}" --build-arg REMOTE_VLLM=0 \
--build-arg GIT_REPO_CHECK=1 --build-arg USE_SCCACHE=1 \
--build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
--build-arg SCCACHE_REGION_NAME=us-west-2 --build-arg SCCACHE_S3_NO_CREDENTIALS=0 .
mkdir -p "$${ROCM_STACK_VLLM_WHEEL_DIR}"
cp rocm-dist/*.whl "$${ROCM_STACK_VLLM_WHEEL_DIR}/"
buildkite-agent artifact upload "$${ROCM_STACK_VLLM_WHEEL_DIR}/*.whl"
- label: ":s3: Upload ROCm 10.0 Wheels to S3"
id: upload-rocm100-wheels
depends_on: build-rocm100-vllm-wheel
agents: *rocm_build_agents
timeout_in_minutes: 60
env: *rocm100_env
commands: &rocm_upload_commands
- |
set -euo pipefail
source .buildkite/scripts/rocm/stack.sh "$${ROCM_BASE_DOCKERFILE}"
buildkite-agent artifact download "$${ROCM_STACK_BASE_WHEELS_DIR}/*.whl" .
buildkite-agent artifact download "$${ROCM_STACK_VLLM_WHEEL_DIR}/*.whl" .
bash .buildkite/scripts/upload-rocm-wheels.sh "$${ROCM_BASE_DOCKERFILE}"
- label: ":memo: Annotate ROCm 10.0 wheel release"
id: annotate-rocm100-release
depends_on: upload-rocm100-wheels
agents: *rocm_build_agents
env: *rocm100_env
commands: &rocm_annotate_commands
- bash .buildkite/scripts/annotate-rocm-release.sh "$${ROCM_BASE_DOCKERFILE}" $${ROCM_DEFAULT_ARG}
# Generate https://wheels.vllm.ai/rocm/ (the default stack's variant), so users can
# `uv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm/` (release only)
- block: "Generate Root Index for ROCm Wheels for Release"
key: block-generate-root-index-rocm-wheels
depends_on:
- upload-rocm100-wheels
- upload-rocm72-wheels
- label: ":package: Generate Root Index for ROCm Wheels for Release"
depends_on: block-generate-root-index-rocm-wheels
id: generate-root-index-rocm-wheels
agents: *rocm_build_agents
env: *rocm100_env
commands:
- "bash tools/vllm-rocm/generate-rocm-wheels-root-index.sh"
- label: ":docker: Build release image - x86_64 - ROCm 10.0"
id: build-rocm100-release-image
depends_on:
- step: block-build-release-images
allow_failure: true
- step: build-rock-base-image
- step: build-rocm100-base
allow_failure: false
agents:
queue: cpu_queue_release
agents: *rocm_build_agents
timeout_in_minutes: 180
commands:
env: *rocm100_env
commands: &rocm_release_image_commands
- |
set -euo pipefail
# Login to ECR
source .buildkite/scripts/rocm/stack.sh "$${ROCM_BASE_DOCKERFILE}"
aws ecr-public get-login-password --region us-east-1 | \
docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rock-base-image-tag 2>/dev/null || echo '')"
if [ -z "$${ECR_IMAGE_TAG}" ]; then
echo "ERROR: rock-base-image-tag metadata not found"
echo "This should have been set by the build-rock-base-image job"
exit 1
fi
docker pull "$${ECR_IMAGE_TAG}"
# Pass the base image ECR tag to downstream steps (nightly publish)
buildkite-agent meta-data set "rock-base-ecr-tag" "$${ECR_IMAGE_TAG}"
echo "========================================"
echo "Building vLLM ROCk release image with:"
echo " BASE_IMAGE: $${ECR_IMAGE_TAG}"
echo " BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}"
echo "========================================"
DOCKER_BUILDKIT=1 docker build \
--build-arg max_jobs=16 \
--build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \
--build-arg USE_SCCACHE=1 \
docker pull "$${ROCM_STACK_ECR_BASE}"
docker build --file "$${ROCM_STACK_DOCKERFILE}" --target vllm-openai \
--tag "$${ROCM_STACK_ECR_IMAGE}" --progress plain \
--build-arg max_jobs=16 --build-arg BASE_IMAGE="$${ROCM_STACK_ECR_BASE}" \
--build-arg INSTALL_LMCACHE=true --build-arg USE_SCCACHE=1 \
--build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
--build-arg SCCACHE_REGION_NAME=us-west-2 \
--build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
--build-arg INSTALL_LMCACHE=true \
--tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rock \
--target vllm-openai \
--progress plain \
-f docker/Dockerfile.rock .
--build-arg SCCACHE_REGION_NAME=us-west-2 --build-arg SCCACHE_S3_NO_CREDENTIALS=0 .
docker push "$${ROCM_STACK_ECR_IMAGE}"
docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rock
env:
- label: "Publish nightly ROCm 10.0 image to DockerHub"
id: publish-rocm100-nightly
depends_on: build-rocm100-release-image
if: build.env("NIGHTLY") == "1"
agents: &rocm_publish_agents
queue: small_cpu_queue_release
env: *rocm100_env
commands: &rocm_publish_nightly_commands
- bash .buildkite/scripts/push-nightly-builds-rocm.sh "$${ROCM_BASE_DOCKERFILE}" $${ROCM_DEFAULT_ARG}
plugins: &rocm_dockerhub_login
- docker-login#v3.0.0:
username: vllmbot
password-env: DOCKERHUB_TOKEN
- group: "Build ROCm 7.2 Wheel / Image"
key: "build-rocm72"
depends_on: ~
steps:
- label: ":rocm: Build ROCm 7.2 Base Image & Wheels"
id: build-rocm72-base
depends_on: ~
agents: *rocm_build_agents
timeout_in_minutes: 360
env: &rocm72_env
ROCM_BASE_DOCKERFILE: "docker/Dockerfile.rocm_72_base"
ROCM_DEFAULT_ARG: ""
DOCKER_BUILDKIT: "1"
S3_BUCKET: "vllm-wheels"
DOCKERHUB_USERNAME: "vllmbot"
commands: *rocm_base_commands
- label: "Publish nightly ROCk image to DockerHub"
- label: ":python: Build vLLM ROCm 7.2 Wheel - x86_64"
id: build-rocm72-vllm-wheel
depends_on: build-rocm72-base
agents: *rocm_build_agents
timeout_in_minutes: 180
env: *rocm72_env
commands: *rocm_vllm_wheel_commands
- label: ":s3: Upload ROCm 7.2 Wheels to S3"
id: upload-rocm72-wheels
depends_on: build-rocm72-vllm-wheel
agents: *rocm_build_agents
timeout_in_minutes: 60
env: *rocm72_env
commands: *rocm_upload_commands
- label: ":memo: Annotate ROCm 7.2 wheel release"
id: annotate-rocm72-release
depends_on: upload-rocm72-wheels
agents: *rocm_build_agents
env: *rocm72_env
commands: *rocm_annotate_commands
- label: ":docker: Build release image - x86_64 - ROCm 7.2"
id: build-rocm72-release-image
depends_on:
- build-rock-release-image
- step: block-build-release-images
allow_failure: true
- step: build-rocm72-base
allow_failure: false
agents: *rocm_build_agents
timeout_in_minutes: 180
env: *rocm72_env
commands: *rocm_release_image_commands
- label: "Publish nightly ROCm 7.2 image to DockerHub"
id: publish-rocm72-nightly
depends_on: build-rocm72-release-image
if: build.env("NIGHTLY") == "1"
agents: *rocm_publish_agents
env: *rocm72_env
commands: *rocm_publish_nightly_commands
plugins: *rocm_dockerhub_login
- label: "Publish nightly XPU image to DockerHub"
depends_on:
- create-manifest-xpu
if: build.env("NIGHTLY") == "1"
agents:
queue: small_cpu_queue_release
commands:
- "bash .buildkite/scripts/push-nightly-builds-rocm.sh rock rocm100"
# Clean up old nightly builds (keep only last 14)
- "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly-rocm100- vllm/vllm-openai-rocm"
- "bash .buildkite/scripts/cleanup-nightly-builds.sh base-nightly-rocm100- vllm/vllm-openai-rocm"
- "bash .buildkite/scripts/xpu/push-nightly-builds-xpu.sh"
- "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly- vllm/vllm-openai-xpu"
plugins:
- docker-login#v3.0.0:
username: vllmbot
@@ -1264,7 +997,8 @@ steps:
- create-multi-arch-manifest-ubuntu2404
- create-multi-arch-manifest-cuda-12-9-ubuntu2404
- create-manifest-xpu
- build-rocm-release-image
- build-rocm100-release-image
- build-rocm72-release-image
- input-release-version
# Wait for CPU builds if their block steps were unblocked, so publish
# doesn't race the in-progress CPU build. allow_failure lets publish
@@ -1289,6 +1023,7 @@ steps:
- "cuda-13-0-ubuntu-24-04"
- "cuda-12-9-ubuntu-24-04"
- "rocm"
- "rocm72"
- "xpu"
- "cpu"
plugins:
.buildkite/scripts/annotate-rocm-release.sh
+40 -66
View File
@@ -2,15 +2,23 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Generate Buildkite annotation for ROCm wheel release
# Generate Buildkite annotation for a ROCm stack's wheel release.
#
# Usage: annotate-rocm-release.sh <base-dockerfile> [--default]
# base-dockerfile e.g. docker/Dockerfile.rocm_base or docker/Dockerfile.rocm_72_base
# --default the stack also owns the plain :latest / :v<ver> Docker Hub tags
set -ex
# Extract build configuration from Dockerfile.rocm_base (single source of truth)
# Extract ROCm version dynamically from Dockerfile.rocm_base
# BASE_IMAGE format: rocm/dev-ubuntu-22.04:7.0-complete -> extracts "7.0"
ROCM_VERSION=$(grep -E '^ARG BASE_IMAGE=' docker/Dockerfile.rocm_base | sed -E 's/.*:([0-9]+\.[0-9]+).*/\1/' || echo "unknown")
PYTHON_VERSION=$(grep '^ARG PYTHON_VERSION=' docker/Dockerfile.rocm_base | sed 's/^ARG PYTHON_VERSION=//')
PYTORCH_ROCM_ARCH=$(grep '^ARG PYTORCH_ROCM_ARCH=' docker/Dockerfile.rocm_base | sed 's/^ARG PYTORCH_ROCM_ARCH=//')
# shellcheck source=.buildkite/scripts/rocm/stack.sh
source "$(dirname "${BASH_SOURCE[0]}")/rocm/stack.sh" "${1:?Usage: $0 <base-dockerfile> [--default]}"
TAG_VARIANTS=("-${ROCM_STACK_VARIANT}")
if [[ "${2:-}" == "--default" ]]; then
TAG_VARIANTS=("" "-${ROCM_STACK_VARIANT}")
fi
ROCM_VERSION="$ROCM_STACK_VERSION"
VARIANT="$ROCM_STACK_VARIANT"
PYTHON_VERSION=$(sed -nE 's/^ARG PYTHON_VERSION="?([^" ]+)"?.*/\1/p' "$ROCM_STACK_BASE_DOCKERFILE")
PYTORCH_ROCM_ARCH=$(sed -nE 's/^ARG PYTORCH_ROCM_ARCH="?([^"]+)"?.*/\1/p' "$ROCM_STACK_BASE_DOCKERFILE")
# Get release version, default to 1.0.0.dev for nightly/per-commit builds
RELEASE_VERSION=$(buildkite-agent meta-data get release-version 2>/dev/null || echo "")
@@ -18,22 +26,32 @@ if [ -z "${RELEASE_VERSION}" ]; then
RELEASE_VERSION="1.0.0.dev"
fi
ROCM_BASE_CACHE_KEY=$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
# S3 URLs
S3_BUCKET="${S3_BUCKET:-vllm-wheels}"
S3_REGION="${AWS_DEFAULT_REGION:-us-west-2}"
S3_URL="http://${S3_BUCKET}.s3-website-${S3_REGION}.amazonaws.com"
S3_HOST="${S3_BUCKET}.s3-website-${S3_REGION}.amazonaws.com"
S3_URL="http://${S3_HOST}"
WHEEL_PATH="rocm/${BUILDKITE_COMMIT}/${VARIANT}-wheels/"
# Format ROCm version for path (e.g., "7.1" -> "rocm710")
ROCM_VERSION_PATH="rocm$(echo "${ROCM_VERSION}" | tr -d '.')"
ROCM_PATH="rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}"
buildkite-agent annotate --style 'success' --context 'rocm-release-workflow' << EOF
## ROCm Wheel and Docker Image Releases
DOCKER_CMDS="docker pull ${ROCM_STACK_ECR_BASE}
docker pull ${ROCM_STACK_ECR_IMAGE}"
for suffix in "${TAG_VARIANTS[@]}"; do
for tag in "latest${suffix}" "v${RELEASE_VERSION}${suffix}"; do
DOCKER_CMDS+="
docker tag ${ROCM_STACK_ECR_BASE} vllm/vllm-openai-rocm:${tag}-base
docker tag ${ROCM_STACK_ECR_IMAGE} vllm/vllm-openai-rocm:${tag}
docker push vllm/vllm-openai-rocm:${tag}-base
docker push vllm/vllm-openai-rocm:${tag}"
done
done
buildkite-agent annotate --style 'success' --context "rocm-release-workflow-${VARIANT}" << EOF
## ROCm ${ROCM_VERSION} Wheel and Docker Image Releases
### Build Configuration
| Setting | Value |
|---------|-------|
| **ROCm Version** | ${ROCM_VERSION} |
| **Index Variant** | ${VARIANT} |
| **Python Version** | ${PYTHON_VERSION} |
| **GPU Architectures** | ${PYTORCH_ROCM_ARCH} |
| **Branch** | \`${BUILDKITE_BRANCH}\` |
@@ -41,49 +59,20 @@ buildkite-agent annotate --style 'success' --context 'rocm-release-workflow' <<
### :package: Installation
**Install from this build (by commit):**
\`\`\`bash
pip install vllm --extra-index-url ${S3_URL}/${ROCM_PATH}/ --trusted-host ${S3_BUCKET}.s3-website-${S3_REGION}.amazonaws.com
# Example for ROCm ${ROCM_VERSION}:
pip install vllm --extra-index-url ${S3_URL}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/ --trusted-host ${S3_BUCKET}.s3-website-${S3_REGION}.amazonaws.com
\`\`\`
**Install from nightly (if published):**
\`\`\`bash
pip install vllm --extra-index-url ${S3_URL}/rocm/nightly/ --trusted-host ${S3_BUCKET}.s3-website-${S3_REGION}.amazonaws.com
# This build (by commit)
pip install vllm --extra-index-url ${S3_URL}/rocm/${BUILDKITE_COMMIT}/${VARIANT}/ --trusted-host ${S3_HOST}
# Nightly (if published)
pip install vllm --extra-index-url ${S3_URL}/rocm/nightly/${VARIANT}/ --trusted-host ${S3_HOST}
\`\`\`
### :floppy_disk: Download Wheels Directly
\`\`\`bash
# List all ROCm wheels
aws s3 ls s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/
# Download specific wheels
aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/vllm-*.whl .
aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/torch-*.whl .
aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/triton-*.whl .
aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/triton-kernels-*.whl .
aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/torchvision-*.whl .
aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/torchaudio-*.whl .
aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/amdsmi-*.whl .
aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/amd_aiter-*.whl .
aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/flash-attn-*.whl .
aws s3 ls s3://${S3_BUCKET}/${WHEEL_PATH}
aws s3 cp --recursive --exclude "*" --include "*.whl" s3://${S3_BUCKET}/${WHEEL_PATH} .
\`\`\`
### :gear: Included Packages
- **vllm**: vLLM with ROCm support
- **torch**: PyTorch built for ROCm ${ROCM_VERSION}
- **triton**: Triton
- **triton-kernels**: Triton kernels
- **torchvision**: TorchVision for ROCm PyTorch
- **torchaudio**: Torchaudio for ROCm PyTorch
- **amdsmi**: AMD SMI Python bindings
- **amd_aiter**: Aiter for ROCm
- **flash-attn**: Flash Attention for ROCm
### :warning: Notes
- These wheels are built for **ROCm ${ROCM_VERSION}** and will NOT work with CUDA GPUs
- Supported GPU architectures: ${PYTORCH_ROCM_ARCH}
@@ -91,23 +80,8 @@ aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/flash-
### :package: Docker Image Release
To download and upload the image:
\`\`\`
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm-base
docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base
docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base vllm/vllm-openai-rocm:latest-base
docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base
docker push vllm/vllm-openai-rocm:latest-base
docker push vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base
docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}
docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT} vllm/vllm-openai-rocm:latest
docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT} vllm/vllm-openai-rocm:v${RELEASE_VERSION}
docker push vllm/vllm-openai-rocm:latest
docker push vllm/vllm-openai-rocm:v${RELEASE_VERSION}
${DOCKER_CMDS}
\`\`\`
EOF
.buildkite/scripts/cache-rocm-base-wheels.sh
+28 -34
View File
@@ -5,39 +5,33 @@
# Cache helper for ROCm base wheels
#
# This script manages caching of pre-built ROCm base wheels (torch, triton, etc.)
# to avoid rebuilding them when Dockerfile.rocm_base hasn't changed.
# to avoid rebuilding them when the stack's base Dockerfile hasn't changed.
#
# Usage:
# cache-rocm-base-wheels.sh check - Check if cache exists, outputs "hit" or "miss"
# cache-rocm-base-wheels.sh upload - Upload wheels to cache
# cache-rocm-base-wheels.sh download - Download wheels from cache
# cache-rocm-base-wheels.sh key - Output the cache key
# Usage: cache-rocm-base-wheels.sh <base-dockerfile> <command>
# base-dockerfile e.g. docker/Dockerfile.rocm_base or docker/Dockerfile.rocm_72_base
# Commands:
# check - Check if cache exists, outputs "hit" or "miss"
# upload - Upload wheels to cache
# download - Download wheels from cache
# key - Output the cache key
# path - Output the full S3 cache path
#
# Environment variables:
# S3_BUCKET - S3 bucket name (default: vllm-wheels)
#
# Note: ROCm version is determined by BASE_IMAGE in Dockerfile.rocm_base,
# so changes to ROCm version are captured by the Dockerfile hash.
# The cache key hashes the base Dockerfile and the repo files it COPYs in, which
# capture the ROCm version and every component pin (see rocm/stack.sh).
set -euo pipefail
BUCKET="${S3_BUCKET:-vllm-wheels}"
DOCKERFILE="docker/Dockerfile.rocm_base"
CACHE_PREFIX="rocm/cache"
# Generate hash from Dockerfile content + build args
generate_cache_key() {
# Include Dockerfile content
if [[ ! -f "$DOCKERFILE" ]]; then
echo "ERROR: Dockerfile not found: $DOCKERFILE" >&2
exit 1
fi
local dockerfile_hash=$(sha256sum "$DOCKERFILE" | cut -c1-16)
echo "${dockerfile_hash}"
}
CACHE_KEY=$(generate_cache_key)
# shellcheck source=.buildkite/scripts/rocm/stack.sh
source "$(dirname "${BASH_SOURCE[0]}")/rocm/stack.sh" "${1:?Usage: $0 <base-dockerfile> <command>}"
shift
CACHE_KEY="$ROCM_STACK_WHEEL_KEY"
WHEELS_DIR="$ROCM_STACK_BASE_WHEELS_DIR"
CACHE_PATH="s3://${BUCKET}/${CACHE_PREFIX}/${CACHE_KEY}/"
case "${1:-}" in
@@ -67,19 +61,19 @@ case "${1:-}" in
echo "Cache path: ${CACHE_PATH}"
echo ""
if [[ ! -d "artifacts/rocm-base-wheels" ]]; then
echo "ERROR: artifacts/rocm-base-wheels directory not found" >&2
if [[ ! -d "${WHEELS_DIR}" ]]; then
echo "ERROR: ${WHEELS_DIR} directory not found" >&2
exit 1
fi
WHEEL_COUNT=$(find artifacts/rocm-base-wheels -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l)
WHEEL_COUNT=$(find "${WHEELS_DIR}" -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l)
if [[ "$WHEEL_COUNT" -eq 0 ]]; then
echo "ERROR: No wheels found in artifacts/rocm-base-wheels/" >&2
echo "ERROR: No wheels found in ${WHEELS_DIR}/" >&2
exit 1
fi
echo "Uploading $WHEEL_COUNT wheels..."
aws s3 cp --recursive artifacts/rocm-base-wheels/ "${CACHE_PATH}"
aws s3 cp --recursive "${WHEELS_DIR}/" "${CACHE_PATH}"
echo ""
echo "Cache upload complete!"
@@ -93,17 +87,17 @@ case "${1:-}" in
echo "Cache key: ${CACHE_KEY}"
echo "Cache path: ${CACHE_PATH}"
echo ""
mkdir -p artifacts/rocm-base-wheels
mkdir -p "${WHEELS_DIR}"
# Use sync with include/exclude to only download .whl files
aws s3 sync "${CACHE_PATH}" artifacts/rocm-base-wheels/ \
aws s3 sync "${CACHE_PATH}" "${WHEELS_DIR}/" \
--exclude "*" \
--include "*.whl"
echo ""
echo "Downloaded wheels:"
find artifacts/rocm-base-wheels -maxdepth 1 -name '*.whl' -exec ls -lh {} \;
WHEEL_COUNT=$(find artifacts/rocm-base-wheels -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l)
find "${WHEELS_DIR}" -maxdepth 1 -name '*.whl' -exec ls -lh {} \;
WHEEL_COUNT=$(find "${WHEELS_DIR}" -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l)
echo ""
echo "Total: $WHEEL_COUNT wheels"
echo "========================================"
@@ -118,12 +112,12 @@ case "${1:-}" in
;;
*)
echo "Usage: $0 {check|upload|download|key|path}" >&2
echo "Usage: $0 <base-dockerfile> {check|upload|download|key|path}" >&2
echo "" >&2
echo "Commands:" >&2
echo " check - Check if cache exists, outputs 'hit' or 'miss'" >&2
echo " upload - Upload wheels from artifacts/rocm-base-wheels/ to cache" >&2
echo " download - Download wheels from cache to artifacts/rocm-base-wheels/" >&2
echo " upload - Upload wheels from ${WHEELS_DIR}/ to cache" >&2
echo " download - Download wheels from cache to ${WHEELS_DIR}/" >&2
echo " key - Output the cache key" >&2
echo " path - Output the full S3 cache path" >&2
exit 1
.buildkite/scripts/ci-bake-rocm.sh
+7 -54
View File
@@ -17,7 +17,7 @@ DEFAULT_REPO_SLUG="vllm-project/vllm"
DEFAULT_CI_HCL_SOURCE="docker/ci-rocm.hcl"
DEFAULT_CI_BASE_CONTENT_FILES=".dockerignore requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt tools/install_torchcodec_rocm.sh rust-toolchain.toml tests/vllm_test_utils"
DEFAULT_CI_BASE_DOCKERFILE="docker/Dockerfile.rocm"
DEFAULT_CI_BASE_DOCKERFILE_STAGES="base rust_toolchain_input_0 rust-toolchain-input rust-toolchain build_nixl lmcache_source build_lmcache build_rocshmem build_deepep mori_base ci_base"
DEFAULT_CI_BASE_DOCKERFILE_STAGES="base rust_toolchain_input_0 rust-toolchain-input rust-toolchain build_nixl lmcache_source build_lmcache build_deepep mori_base ci_base"
DEFAULT_CI_BASE_METADATA_VERSION="3"
# ROCm CI forces REMOTE_VLLM=0, so content identity covers only the selected
# local-source stages rather than unreachable remote-fetch alternatives.
@@ -555,9 +555,6 @@ configure_custom_rocm_stages() {
CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${stages}}"
ROCM_CSRC_DOCKERFILE_STAGES="${ROCM_CSRC_DOCKERFILE_STAGES:-${stages}}"
ROCM_RUST_DOCKERFILE_STAGES="${ROCM_RUST_DOCKERFILE_STAGES:-${stages}}"
if [[ " ${stages} " != *" build_rocshmem "* ]]; then
unset ROCSHMEM_BRANCH ROCSHMEM_CACHE_KEY DEEPEP_CACHE_KEY
fi
}
compose_dependency_cache_key() {
@@ -1724,14 +1721,11 @@ ci_base_metadata_pairs() {
metadata_pair "vllm.rocm.nixl_commit" "${NIXL_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "NIXL_BRANCH")}"
metadata_pair "vllm.rocm.ucx_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "UCX_REPO")"
metadata_pair "vllm.rocm.ucx_commit" "${UCX_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "UCX_BRANCH")}"
metadata_pair "vllm.rocm.rocshmem_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "ROCSHMEM_REPO")"
metadata_pair "vllm.rocm.rocshmem_commit" "${ROCSHMEM_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "ROCSHMEM_BRANCH")}"
metadata_pair "vllm.rocm.deepep_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_REPO")"
metadata_pair "vllm.rocm.deepep_commit" "${DEEPEP_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_BRANCH")}"
metadata_pair "vllm.rocm.deepep_nic" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_NIC")"
metadata_pair "vllm.rocm.deepep_rocm_arch" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_ROCM_ARCH")"
metadata_pair "vllm.rocm.nixl_cache_key" "${NIXL_CACHE_KEY:-}"
metadata_pair "vllm.rocm.rocshmem_cache_key" "${ROCSHMEM_CACHE_KEY:-}"
metadata_pair "vllm.rocm.deepep_cache_key" "${DEEPEP_CACHE_KEY:-}"
}
@@ -2314,7 +2308,7 @@ extract_dependency_pins() {
return 0
fi
for var in NIXL_BRANCH UCX_BRANCH ROCSHMEM_BRANCH DEEPEP_BRANCH; do
for var in NIXL_BRANCH UCX_BRANCH DEEPEP_BRANCH; do
if [[ -n "${!var:-}" ]]; then
echo "Using provided ${var}: ${!var}"
continue
@@ -2336,17 +2330,14 @@ compute_dependency_cache_keys() {
local dockerfile_rocm=""
local nixl_branch=""
local ucx_branch=""
local rocshmem_branch=""
local deepep_branch=""
local nixl_material=""
local rocshmem_material=""
local deepep_material=""
bake_dir=$(dirname "${VLLM_BAKE_FILE}")
dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
nixl_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "NIXL_BRANCH")
ucx_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "UCX_BRANCH")
rocshmem_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "ROCSHMEM_BRANCH")
deepep_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "DEEPEP_BRANCH")
if [[ -n "${nixl_branch}" && -n "${ucx_branch}" ]]; then
@@ -2360,22 +2351,11 @@ compute_dependency_cache_keys() {
echo "NIXL dependency cache key: ${NIXL_CACHE_KEY}"
fi
if [[ -n "${rocshmem_branch}" ]]; then
rocshmem_material=$(compose_stage_cache_material "${dockerfile_rocm}" "base build_rocshmem")
ROCSHMEM_CACHE_KEY=$(
compose_dependency_cache_key \
"${rocshmem_branch}" \
"${rocshmem_material}"
)
export ROCSHMEM_CACHE_KEY
echo "ROCShmem dependency cache key: ${ROCSHMEM_CACHE_KEY}"
fi
if [[ -n "${deepep_branch}" && -n "${rocshmem_branch}" ]]; then
deepep_material=$(compose_stage_cache_material "${dockerfile_rocm}" "base build_rocshmem build_deepep")
if [[ -n "${deepep_branch}" ]]; then
deepep_material=$(compose_stage_cache_material "${dockerfile_rocm}" "base build_deepep")
DEEPEP_CACHE_KEY=$(
compose_dependency_cache_key \
"${deepep_branch}-rocshmem-${rocshmem_branch}" \
"${deepep_branch}" \
"${deepep_material}"
)
export DEEPEP_CACHE_KEY
@@ -2415,18 +2395,11 @@ dependency_cache_ref_for_target() {
printf '%s\n' "${cache_repo}:nixl-rocm-${NIXL_BRANCH}-ucx-${UCX_BRANCH:-}"
fi
;;
rocshmem-rocm-ci)
if [[ -n "${ROCSHMEM_CACHE_KEY:-}" ]]; then
printf '%s\n' "${cache_repo}:rocshmem-rocm-${ROCSHMEM_CACHE_KEY}"
elif [[ -n "${ROCSHMEM_BRANCH:-}" ]]; then
printf '%s\n' "${cache_repo}:rocshmem-rocm-${ROCSHMEM_BRANCH}"
fi
;;
deepep-rocm-ci)
if [[ -n "${DEEPEP_CACHE_KEY:-}" ]]; then
printf '%s\n' "${cache_repo}:deepep-rocm-${DEEPEP_CACHE_KEY}"
elif [[ -n "${DEEPEP_BRANCH:-}" ]]; then
printf '%s\n' "${cache_repo}:deepep-rocm-${DEEPEP_BRANCH}-rocshmem-${ROCSHMEM_BRANCH:-}"
printf '%s\n' "${cache_repo}:deepep-rocm-${DEEPEP_BRANCH}"
fi
;;
esac
@@ -2444,7 +2417,6 @@ add_dependency_cache_target() {
resolve_ci_base_dependency_targets() {
local mode="${ROCM_DEP_CACHE_EXPORT_MODE:-missing}"
local nixl_ref=""
local rocshmem_ref=""
local deepep_ref=""
[[ "${TARGET}" == "ci-base-rocm-ci-with-deps" ]] || return 0
@@ -2452,7 +2424,7 @@ resolve_ci_base_dependency_targets() {
case "${mode}" in
always)
echo "ROCM_DEP_CACHE_EXPORT_MODE=always; exporting all dependency caches serially"
for target in nixl-rocm-ci rocshmem-rocm-ci deepep-rocm-ci; do
for target in nixl-rocm-ci deepep-rocm-ci; do
if [[ -n "$(dependency_cache_ref_for_target "${target}")" ]]; then
add_dependency_cache_target "${target}"
fi
@@ -2482,16 +2454,6 @@ resolve_ci_base_dependency_targets() {
fi
fi
if [[ "${mode}" != "always" && -n "${ROCSHMEM_CACHE_KEY:-}" ]]; then
rocshmem_ref=$(dependency_cache_ref_for_target "rocshmem-rocm-ci")
if dependency_cache_ref_exists "${rocshmem_ref}"; then
echo "ROCShmem dependency cache exists: ${rocshmem_ref}"
else
echo "ROCShmem dependency cache missing; will seed: ${rocshmem_ref}"
add_dependency_cache_target "rocshmem-rocm-ci"
fi
fi
if [[ "${mode}" != "always" && -n "${DEEPEP_CACHE_KEY:-}" ]]; then
deepep_ref=$(dependency_cache_ref_for_target "deepep-rocm-ci")
if dependency_cache_ref_exists "${deepep_ref}"; then
@@ -2502,15 +2464,6 @@ resolve_ci_base_dependency_targets() {
fi
fi
# DeepEP inherits from ROCShmem. If ROCShmem is being seeded, seed DeepEP too
# so the pair stays consistent for future ci_base rebuilds.
if printf '%s\n' "${DEPENDENCY_CACHE_TARGETS[@]}" | grep -qx "rocshmem-rocm-ci" \
&& ! printf '%s\n' "${DEPENDENCY_CACHE_TARGETS[@]}" | grep -qx "deepep-rocm-ci" \
&& [[ -n "${DEEPEP_BRANCH:-}" ]]; then
echo "ROCShmem cache is missing; also seeding DeepEP cache"
add_dependency_cache_target "deepep-rocm-ci"
fi
BAKE_TARGETS=("ci-base-rocm-ci")
if [[ ${#DEPENDENCY_CACHE_TARGETS[@]} -eq 0 ]]; then
echo "All dependency caches exist; building ci_base without dependency cache exports"
.buildkite/scripts/cleanup-nightly-builds.sh
+5 -2
View File
@@ -5,14 +5,17 @@ set -ex
# Clean up old nightly builds from DockerHub, keeping only the last 14 builds
# This script uses DockerHub API to list and delete old tags with specified prefix
# Tags starting with "nightly-dev" are always excluded from deletion
# Usage: cleanup-nightly-builds.sh [TAG_PREFIX] [REPO]
# Usage: cleanup-nightly-builds.sh [TAG_PREFIX] [REPO] [EXCLUDE_PREFIX]
# Example: cleanup-nightly-builds.sh "nightly-"
# Example: cleanup-nightly-builds.sh "cu130-nightly-"
# Example: cleanup-nightly-builds.sh "nightly-" "vllm/vllm-openai-rocm"
# Example: cleanup-nightly-builds.sh "nightly-" "vllm/vllm-openai-rocm" "nightly-rocm"
# (EXCLUDE_PREFIX skips tags owned by another variant sharing TAG_PREFIX)
# Get tag prefix and repo from arguments
TAG_PREFIX="${1:-nightly-}"
REPO="${2:-vllm/vllm-openai}"
EXCLUDE_PREFIX="${3:-}"
echo "Cleaning up tags with prefix: $TAG_PREFIX in repository: $REPO"
@@ -57,7 +60,7 @@ get_all_tags() {
# Get both last_updated timestamp and tag name, separated by |
# Exclude mutable architecture tags and nightly-dev tags from cleanup
local tags=$(echo "$response" | jq -r --arg prefix "$TAG_PREFIX" '.results[] | select(.name | startswith($prefix)) | select(.name != ($prefix + "x86_64") and .name != ($prefix + "aarch64")) | select(.name | startswith("nightly-dev") | not) | "\(.last_updated)|\(.name)"')
local tags=$(echo "$response" | jq -r --arg prefix "$TAG_PREFIX" --arg exclude "$EXCLUDE_PREFIX" '.results[] | select(.name | startswith($prefix)) | select($exclude == "" or (.name | startswith($exclude) | not)) | select(.name != ($prefix + "x86_64") and .name != ($prefix + "aarch64")) | select(.name | startswith("nightly-dev") | not) | "\(.last_updated)|\(.name)"')
if [ -z "$tags" ]; then
break
.buildkite/scripts/generate-nightly-index.py
+63 -9
View File
@@ -12,7 +12,7 @@ from dataclasses import asdict, dataclass
from datetime import datetime
from pathlib import Path
from typing import Any
from urllib.parse import quote
from urllib.parse import quote, unquote, urlparse
import regex as re
@@ -51,6 +51,8 @@ class WheelFileInfo:
platform_tag: str
variant: str | None
filename: str
# Absolute URL for files hosted elsewhere (see --external-links)
url: str | None = None
def parse_from_filename(file: str) -> WheelFileInfo:
@@ -110,6 +112,29 @@ def parse_from_filename(file: str) -> WheelFileInfo:
)
def parse_external_url(url: str) -> WheelFileInfo:
"""Parse an absolute wheel or sdist URL hosted on another index."""
filename = unquote(urlparse(url).path.rsplit("/", 1)[-1])
if filename.endswith(".whl"):
info = parse_from_filename(filename)
else:
match = re.match(r"^(?P<name>.+)-(?P<version>[^-]+)\.(tar\.gz|zip)$", filename)
if not match:
raise ValueError(f"Unsupported external file: {filename}")
info = WheelFileInfo(
package_name=match.group("name"),
version=match.group("version"),
build_tag=None,
python_tag="source",
abi_tag="none",
platform_tag="any",
variant=None,
filename=filename,
)
info.url = url
return info
def generate_project_list(subdir_names: list[str], comment: str = "") -> str:
"""Generate project list HTML content linking to each project & variant subdirectory."""
href_tags = []
@@ -129,14 +154,19 @@ def generate_package_index_and_metadata(
href_tags = []
metadata = []
for file in sorted(wheel_files, key=lambda x: x.filename):
relative_path = (
wheel_base_dir.relative_to(index_base_dir, walk_up=True) / file.filename
)
# handle with '+' in URL, and avoid double-encoding '/' and already-encoded '%2B'
# NOTE: this is AWS S3 specific behavior!
file_path_quoted = quote(relative_path.as_posix(), safe=":%/")
if file.url:
file_path_quoted = file.url
else:
relative_path = (
wheel_base_dir.relative_to(index_base_dir, walk_up=True) / file.filename
)
# handle with '+' in URL, and avoid double-encoding '/' and already-encoded '%2B'
# NOTE: this is AWS S3 specific behavior!
file_path_quoted = quote(relative_path.as_posix(), safe=":%/")
href_tags.append(f' <a href="{file_path_quoted}">{file.filename}</a><br/>')
file_meta = asdict(file)
if file_meta["url"] is None:
del file_meta["url"]
file_meta["path"] = file_path_quoted
metadata.append(file_meta)
index_str = INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags), comment=comment)
@@ -151,6 +181,7 @@ def generate_index_and_metadata(
default_variant: str | None = None,
alias_to_default: str | None = None,
comment: str = "",
external_urls: list[str] | None = None,
):
"""Generate index for all wheel files.
@@ -160,6 +191,8 @@ def generate_index_and_metadata(
index_base_dir (Path): Base directory to store index files.
default_variant (str | None): The default variant name, if any.
alias_to_default (str | None): Alias variant name for the default variant, if any.
external_urls (list[str] | None): Absolute URLs of files hosted on another index,
listed in the generated indices without being copied.
comment (str | None): Optional comment to include in the generated HTML files.
First, parse all wheel files to extract metadata.
@@ -214,6 +247,7 @@ def generate_index_and_metadata(
"""
parsed_files = [parse_from_filename(f) for f in whl_files]
parsed_files += [parse_external_url(u) for u in external_urls or []]
if not parsed_files:
print("No wheel files found, skipping index generation.")
@@ -232,10 +266,14 @@ def generate_index_and_metadata(
print(f"Detected ROCm variant from vllm: {rocm_variant}")
break
# Apply ROCm variant to all wheels without a variant
# Apply ROCm variant to all wheels without a variant. Dependencies with their
# own ROCm local version (e.g. TheRock's torch-2.12.0+rocm10.0.0) belong to
# the same variant rather than one of their own.
if rocm_variant:
for file in parsed_files:
if file.variant is None:
if file.variant is None or (
file.variant.startswith("rocm") and file.variant != rocm_variant
):
file.variant = rocm_variant
print(f"Inherited variant '{rocm_variant}' for {file.filename}")
@@ -366,6 +404,13 @@ if __name__ == "__main__":
default=None,
help="Alias variant name for the default variant",
)
parser.add_argument(
"--external-links",
type=str,
default=None,
help="Optional file with one absolute URL per line for packages hosted on "
"another index (listed alongside, but not copied into, this index)",
)
parser.add_argument(
"--comment",
type=str,
@@ -463,5 +508,14 @@ if __name__ == "__main__":
default_variant=None,
alias_to_default=args.alias_to_default,
comment=args.comment.strip(),
external_urls=(
[
line.strip()
for line in Path(args.external_links).read_text().splitlines()
if line.strip() and not line.startswith("#")
]
if args.external_links
else None
),
)
print(f"Successfully generated index and metadata in {output_dir}")
.buildkite/scripts/publish-release-images.sh
+28 -16
View File
@@ -11,9 +11,9 @@ set -euo pipefail
TARGET="${1:-all}"
case "${TARGET}" in
cuda-13-0 | cuda-12-9 | cuda-13-0-ubuntu-24-04 | \
cuda-12-9-ubuntu-24-04 | rocm | xpu | cpu | all) ;;
cuda-12-9-ubuntu-24-04 | rocm | rocm72 | xpu | cpu | all) ;;
*)
echo "Usage: $0 {cuda-13-0|cuda-12-9|cuda-13-0-ubuntu-24-04|cuda-12-9-ubuntu-24-04|rocm|xpu|cpu|all}"
echo "Usage: $0 {cuda-13-0|cuda-12-9|cuda-13-0-ubuntu-24-04|cuda-12-9-ubuntu-24-04|rocm|rocm72|xpu|cpu|all}"
exit 2
;;
esac
@@ -143,23 +143,35 @@ if target_enabled cuda-12-9-ubuntu-24-04; then
fi
# ---- ROCm ----
# Each ROCm stack is identified by its base Dockerfile (see rocm/stack.sh). The
# default stack also owns the plain :latest / :v<ver> tags.
publish_rocm_stack() {
local base_dockerfile="$1" default="$2" suffix
(
# shellcheck source=.buildkite/scripts/rocm/stack.sh
source .buildkite/scripts/rocm/stack.sh "$base_dockerfile"
local suffixes=("-${ROCM_STACK_VARIANT}")
[[ "$default" == "1" ]] && suffixes=("" "-${ROCM_STACK_VARIANT}")
docker pull "$ROCM_STACK_ECR_IMAGE"
docker pull "$ROCM_STACK_ECR_BASE"
for suffix in "${suffixes[@]}"; do
for tag in "latest${suffix}" "v${RELEASE_VERSION}${suffix}"; do
docker tag "$ROCM_STACK_ECR_IMAGE" "vllm/vllm-openai-rocm:${tag}"
docker tag "$ROCM_STACK_ECR_BASE" "vllm/vllm-openai-rocm:${tag}-base"
docker push "vllm/vllm-openai-rocm:${tag}"
docker push "vllm/vllm-openai-rocm:${tag}-base"
done
done
)
}
if target_enabled rocm; then
ROCM_BASE_CACHE_KEY=$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
echo "ROCm base cache key: ${ROCM_BASE_CACHE_KEY}"
publish_rocm_stack docker/Dockerfile.rocm_base 1
fi
docker pull "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm"
docker pull "public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base"
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm" vllm/vllm-openai-rocm:latest
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm" "vllm/vllm-openai-rocm:v${RELEASE_VERSION}"
docker push vllm/vllm-openai-rocm:latest
docker push "vllm/vllm-openai-rocm:v${RELEASE_VERSION}"
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base" vllm/vllm-openai-rocm:latest-base
docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base" "vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base"
docker push vllm/vllm-openai-rocm:latest-base
docker push "vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base"
if target_enabled rocm72; then
publish_rocm_stack docker/Dockerfile.rocm_72_base 0
fi
# ---- XPU ----
.buildkite/scripts/push-nightly-builds-rocm.sh
+42 -65
View File
@@ -2,42 +2,36 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Push a ROCm-family nightly base image and nightly image from ECR to Docker Hub
# under vllm/vllm-openai-rocm, as base-nightly[-<variant>], nightly[-<variant>]
# and their -<commit> forms.
# Run when NIGHTLY=1 after the matching build-*-release-image step has pushed to ECR.
# Push a ROCm stack's nightly base image and nightly image from ECR to Docker Hub
# under vllm/vllm-openai-rocm, as base-nightly-<variant>, nightly-<variant> and their
# -<commit> forms, then prune old nightly tags. With --default, also push the plain
# base-nightly and nightly tags for the same images.
# Run when NIGHTLY=1 after the stack's release image has been pushed to ECR.
#
# Usage: push-nightly-builds-rocm.sh [ECR_TAG_SUFFIX TAG_VARIANT]
# Pass both arguments or neither. With none, the default ROCm apt build is
# published as :nightly. The TheRock build passes "rock" (the source ECR tag
# suffix) and "rocm714" (the Docker Hub tag flavor), giving :nightly-rocm714.
# Usage: push-nightly-builds-rocm.sh <base-dockerfile> [--default]
# e.g. "docker/Dockerfile.rocm_base --default" (ROCm 10.0, also :nightly)
# and "docker/Dockerfile.rocm_72_base" (ROCm 7.2, :nightly-rocm72).
#
# Local testing (no push to Docker Hub):
# BASE_ECR_IMAGE=<full-base-image-reference> \
# BUILDKITE_COMMIT=<commit-with-image-in-ecr> DRY_RUN=1 \
# bash .buildkite/scripts/push-nightly-builds-rocm.sh
# BUILDKITE_COMMIT=<commit-with-image-in-ecr> DRY_RUN=1 \
# bash .buildkite/scripts/push-nightly-builds-rocm.sh docker/Dockerfile.rocm_base --default
# Requires: AWS CLI configured (for ECR public login), Docker. For full run: Docker Hub login.
set -euxo pipefail
usage() {
echo "Usage: $0 [ECR_TAG_SUFFIX TAG_VARIANT]" >&2
echo "Usage: $0 <base-dockerfile> [--default]" >&2
}
DOCKERHUB_REPO="vllm/vllm-openai-rocm"
PUSH_DEFAULT=0
case "$#" in
0)
ECR_TAG_SUFFIX="rocm"
TAG_VARIANT=""
;;
1) ;;
2)
if [[ -z "$1" || -z "$2" ]]; then
if [[ "$2" != "--default" ]]; then
usage
exit 2
fi
ECR_TAG_SUFFIX="$1"
TAG_VARIANT="$2"
PUSH_DEFAULT=1
;;
*)
usage
@@ -49,61 +43,44 @@ esac
BUILDKITE_COMMIT="${BUILDKITE_COMMIT:?Set BUILDKITE_COMMIT to the commit SHA that has the image in ECR (e.g. from a previous release pipeline run)}"
DRY_RUN="${DRY_RUN:-0}"
ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-release-repo"
# shellcheck source=.buildkite/scripts/rocm/stack.sh
source "$(dirname "${BASH_SOURCE[0]}")/rocm/stack.sh" "$1"
DOCKERHUB_REPO="vllm/vllm-openai-rocm"
VARIANT="$ROCM_STACK_VARIANT"
# Get the base image ECR tag (set by the build-*-release-image pipeline step)
BASE_METADATA_KEY="${ECR_TAG_SUFFIX}-base-ecr-tag"
if [[ -n "${BASE_ECR_IMAGE:-}" ]]; then
if [[ "$DRY_RUN" != "1" ]]; then
echo "ERROR: BASE_ECR_IMAGE may only be used with DRY_RUN=1" >&2
exit 1
fi
BASE_ORIG_TAG="$BASE_ECR_IMAGE"
elif ! BASE_ORIG_TAG="$(buildkite-agent meta-data get "$BASE_METADATA_KEY")"; then
echo "ERROR: Failed to read required Buildkite metadata '$BASE_METADATA_KEY'" >&2
exit 1
fi
if [[ -z "$BASE_ORIG_TAG" ]]; then
echo "ERROR: Required Buildkite metadata '$BASE_METADATA_KEY' is empty" >&2
echo "Set BASE_ECR_IMAGE to the full ECR base image reference for local testing" >&2
exit 1
BASE_TAGS=("base-nightly-${VARIANT}" "base-nightly-${VARIANT}-${BUILDKITE_COMMIT}")
TAGS=("nightly-${VARIANT}" "nightly-${VARIANT}-${BUILDKITE_COMMIT}")
if [[ "$PUSH_DEFAULT" == "1" ]]; then
BASE_TAGS+=("base-nightly" "base-nightly-${BUILDKITE_COMMIT}")
TAGS+=("nightly" "nightly-${BUILDKITE_COMMIT}")
fi
ORIG_TAG="${BUILDKITE_COMMIT}-${ECR_TAG_SUFFIX}"
VARIANT_SUFFIX=""
[[ -n "$TAG_VARIANT" ]] && VARIANT_SUFFIX="-$TAG_VARIANT"
BASE_TAG_NAME="base-nightly${VARIANT_SUFFIX}"
TAG_NAME="nightly${VARIANT_SUFFIX}"
BASE_TAG_NAME_COMMIT="base-nightly${VARIANT_SUFFIX}-${BUILDKITE_COMMIT}"
TAG_NAME_COMMIT="nightly${VARIANT_SUFFIX}-${BUILDKITE_COMMIT}"
echo "Pushing base image $ROCM_STACK_ECR_BASE as: ${BASE_TAGS[*]}"
echo "Pushing release image $ROCM_STACK_ECR_IMAGE as: ${TAGS[*]}"
echo "Pushing base image from ECR: $BASE_ORIG_TAG"
echo "Pushing release image from ECR tag: $ORIG_TAG to Docker Hub as $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT"
[[ "$DRY_RUN" == "1" ]] && echo "[DRY_RUN] Skipping push to Docker Hub"
# Login to ECR and pull the images built by build-*-release-image
# Login to ECR and pull the stack's images
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
docker pull "$BASE_ORIG_TAG"
docker pull "${ECR_REPO}:${ORIG_TAG}"
docker pull "$ROCM_STACK_ECR_BASE"
docker pull "$ROCM_STACK_ECR_IMAGE"
# Tag for Docker Hub (base-nightly and base-nightly-<commit>, nightly and nightly-<commit>)
docker tag "$BASE_ORIG_TAG" "$DOCKERHUB_REPO":"$BASE_TAG_NAME"
docker tag "$BASE_ORIG_TAG" "$DOCKERHUB_REPO":"$BASE_TAG_NAME_COMMIT"
docker tag "${ECR_REPO}:${ORIG_TAG}" "$DOCKERHUB_REPO":"$TAG_NAME"
docker tag "${ECR_REPO}:${ORIG_TAG}" "$DOCKERHUB_REPO":"$TAG_NAME_COMMIT"
for tag in "${BASE_TAGS[@]}"; do docker tag "$ROCM_STACK_ECR_BASE" "$DOCKERHUB_REPO:$tag"; done
for tag in "${TAGS[@]}"; do docker tag "$ROCM_STACK_ECR_IMAGE" "$DOCKERHUB_REPO:$tag"; done
if [[ "$DRY_RUN" == "1" ]]; then
echo "[DRY_RUN] Would push $DOCKERHUB_REPO:$BASE_TAG_NAME and $DOCKERHUB_REPO:$BASE_TAG_NAME_COMMIT"
echo "[DRY_RUN] Would push $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT"
echo "[DRY_RUN] Local tags created. Exiting without push."
exit 0
fi
# Push to Docker Hub (docker-login plugin runs before this step in CI)
docker push "$DOCKERHUB_REPO":"$BASE_TAG_NAME"
docker push "$DOCKERHUB_REPO":"$BASE_TAG_NAME_COMMIT"
docker push "$DOCKERHUB_REPO":"$TAG_NAME"
docker push "$DOCKERHUB_REPO":"$TAG_NAME_COMMIT"
for tag in "${BASE_TAGS[@]}" "${TAGS[@]}"; do docker push "$DOCKERHUB_REPO:$tag"; done
echo "Pushed $DOCKERHUB_REPO: ${BASE_TAGS[*]} ${TAGS[*]}"
echo "Pushed $DOCKERHUB_REPO:$BASE_TAG_NAME and $DOCKERHUB_REPO:$BASE_TAG_NAME_COMMIT"
echo "Pushed $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT"
# Keep only the last 14 builds of each tag family. The plain nightly- families
# exclude the variant ones sharing their prefix; commit SHAs never start with "rocm".
CLEANUP=.buildkite/scripts/cleanup-nightly-builds.sh
bash "$CLEANUP" "nightly-${VARIANT}-" "$DOCKERHUB_REPO"
bash "$CLEANUP" "base-nightly-${VARIANT}-" "$DOCKERHUB_REPO"
if [[ "$PUSH_DEFAULT" == "1" ]]; then
bash "$CLEANUP" "nightly-" "$DOCKERHUB_REPO" "nightly-rocm"
bash "$CLEANUP" "base-nightly-" "$DOCKERHUB_REPO" "base-nightly-rocm"
fi
.buildkite/scripts/rocm/build-config.sh
+2 -2
View File
@@ -1,8 +1,8 @@
#!/usr/bin/env bash
# Shared Dockerfile selection for the normal AMD CI build chain.
# CI_ROCM_DOCKERFILE_BASE and CI_ROCM_DOCKERFILE accept repository-relative paths.
# For Rock, set CI_ROCM_DOCKERFILE_BASE=docker/Dockerfile.rock_base and
# CI_ROCM_DOCKERFILE=docker/Dockerfile.rock.
# For the legacy ROCm 7.2 stack, set CI_ROCM_DOCKERFILE_BASE=docker/Dockerfile.rocm_72_base
# and CI_ROCM_DOCKERFILE=docker/Dockerfile.rocm_72.
# Custom recipes must preserve the CI targets and artifact layout. Additional
# COPY inputs also need the corresponding *_CONTENT_FILES overrides.
.buildkite/scripts/rocm/stack.sh
+78
View File
@@ -0,0 +1,78 @@
#!/usr/bin/env bash
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
#
# Derive everything the release pipeline needs about a ROCm stack from its base
# Dockerfile, so the stack's image is the single input to every ROCm script.
#
# Usage: source .buildkite/scripts/rocm/stack.sh <base-dockerfile>
# e.g. docker/Dockerfile.rocm_base (ROCm 10.0) or docker/Dockerfile.rocm_72_base
#
# Exports:
# ROCM_STACK_BASE_DOCKERFILE the base Dockerfile
# ROCM_STACK_DOCKERFILE the matching vLLM Dockerfile (base name minus _base)
# ROCM_STACK_THEROCK 1 if the ROCm SDK comes from TheRock wheels
# ROCM_STACK_VERSION ROCm version, e.g. 10.0.0 or 7.2.3
# ROCM_STACK_VARIANT wheel variant and Docker Hub tag flavor from major.minor,
# same rule as setup.py (rocm100, rocm72)
# ROCM_STACK_IMAGE_KEY hash of the base Dockerfile (base image cache key)
# ROCM_STACK_WHEEL_KEY hash of the base Dockerfile plus files it COPYs from
# the repo (base wheel cache key)
# ROCM_STACK_ECR_BASE ECR tag of the cached base image
# ROCM_STACK_ECR_IMAGE ECR tag of this commit's release image
# ROCM_STACK_BASE_WHEELS_DIR local dir for base wheels (also the artifact path)
# ROCM_STACK_VLLM_WHEEL_DIR local dir for the vLLM wheel (also the artifact path)
rocm_stack_init() {
local base="$1"
if [[ ! -f "$base" ]]; then
echo "ROCm base Dockerfile not found: $base" >&2
return 2
fi
ROCM_STACK_BASE_DOCKERFILE="$base"
ROCM_STACK_DOCKERFILE="${base%_base}"
local sdk
sdk=$(sed -nE 's/^ARG ROCM_SDK_VERSION=([0-9.]+).*/\1/p' "$base" | head -1)
if [[ -n "$sdk" ]]; then
ROCM_STACK_THEROCK=1
ROCM_STACK_VERSION="$sdk"
else
ROCM_STACK_THEROCK=0
# BASE_IMAGE format: rocm/dev-ubuntu-22.04:7.2.3-complete -> 7.2.3
ROCM_STACK_VERSION=$(sed -nE 's/^ARG BASE_IMAGE=.*:([0-9]+(\.[0-9]+)+).*/\1/p' "$base" | head -1)
fi
if [[ -z "$ROCM_STACK_VERSION" ]]; then
echo "Could not determine the ROCm version from $base" >&2
return 2
fi
local major minor
IFS=. read -r major minor _ <<< "$ROCM_STACK_VERSION"
ROCM_STACK_VARIANT="rocm${major}${minor}"
ROCM_STACK_IMAGE_KEY=$(sha256sum "$base" | cut -c1-16)
local inputs
inputs=$(grep -E '^COPY ' "$base" | grep -v -- '--from' | awk '{print $2}' || true)
if [[ -z "$inputs" ]]; then
ROCM_STACK_WHEEL_KEY="$ROCM_STACK_IMAGE_KEY"
else
# shellcheck disable=SC2086
ROCM_STACK_WHEEL_KEY=$(sha256sum "$base" $inputs | sha256sum | cut -c1-16)
fi
local ecr="public.ecr.aws/q9t5s3a7/vllm-release-repo"
ROCM_STACK_ECR_BASE="${ecr}:${ROCM_STACK_IMAGE_KEY}-${ROCM_STACK_VARIANT}-base"
ROCM_STACK_ECR_IMAGE="${ecr}:${BUILDKITE_COMMIT:-unknown}-${ROCM_STACK_VARIANT}"
ROCM_STACK_BASE_WHEELS_DIR="artifacts/${ROCM_STACK_VARIANT}-base-wheels"
ROCM_STACK_VLLM_WHEEL_DIR="artifacts/${ROCM_STACK_VARIANT}-vllm-wheel"
export ROCM_STACK_BASE_DOCKERFILE ROCM_STACK_DOCKERFILE ROCM_STACK_THEROCK \
ROCM_STACK_VERSION ROCM_STACK_VARIANT ROCM_STACK_IMAGE_KEY \
ROCM_STACK_WHEEL_KEY ROCM_STACK_ECR_BASE ROCM_STACK_ECR_IMAGE \
ROCM_STACK_BASE_WHEELS_DIR ROCM_STACK_VLLM_WHEEL_DIR
}
if [[ $# -gt 0 ]]; then
rocm_stack_init "$1"
fi
.buildkite/scripts/upload-rocm-wheels.sh
+66 -20
View File
@@ -4,21 +4,31 @@
#
# Upload ROCm wheels to S3 with proper index generation
#
# Usage: upload-rocm-wheels.sh <base-dockerfile>
# base-dockerfile the ROCm stack, e.g. docker/Dockerfile.rocm_base (see rocm/stack.sh)
#
# Required environment variables:
# AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY (or IAM role)
# S3_BUCKET (default: vllm-wheels)
#
# S3 path structure:
# s3://vllm-wheels/rocm/{commit}/ - All wheels for this commit
# s3://vllm-wheels/rocm/nightly/ - Index pointing to latest nightly
# s3://vllm-wheels/rocm/{version}/ - Index for release versions
# s3://vllm-wheels/rocm/{commit}/<variant>-wheels/ - This stack's wheels for this commit
# s3://vllm-wheels/rocm/{commit}/<variant>/ - Index per ROCm variant (e.g. rocm100)
# s3://vllm-wheels/rocm/nightly/<variant>/ - Index pointing to latest nightly
# s3://vllm-wheels/rocm/{version}/<variant>/ - Index for release versions
# The top-level index.html at each prefix lists every variant present, since
# several ROCm stacks publish into the same prefixes.
set -ex
# ======== Configuration ========
BUCKET="${S3_BUCKET:-vllm-wheels}"
# shellcheck source=.buildkite/scripts/rocm/stack.sh
source "$(dirname "${BASH_SOURCE[0]}")/rocm/stack.sh" "${1:?Usage: $0 <base-dockerfile>}"
ROCM_SUBPATH="rocm/${BUILDKITE_COMMIT}"
S3_COMMIT_PREFIX="s3://$BUCKET/$ROCM_SUBPATH/"
WHEEL_DIRS="$ROCM_STACK_BASE_WHEELS_DIR $ROCM_STACK_VLLM_WHEEL_DIR"
WHEEL_SUBPATH="$ROCM_SUBPATH/${ROCM_STACK_VARIANT}-wheels"
S3_WHEEL_PREFIX="s3://$BUCKET/$WHEEL_SUBPATH/"
INDICES_OUTPUT_DIR="rocm-indices"
echo "========================================"
@@ -49,9 +59,11 @@ source .buildkite/scripts/lib/manylinux.sh
# ======== Part 1: Collect and prepare wheels ========
# Collect all wheels
rm -rf all-rocm-wheels "$INDICES_OUTPUT_DIR"
mkdir -p all-rocm-wheels
cp artifacts/rocm-base-wheels/*.whl all-rocm-wheels/ 2>/dev/null || true
cp artifacts/rocm-vllm-wheel/*.whl all-rocm-wheels/ 2>/dev/null || true
for dir in $WHEEL_DIRS; do
cp "$dir"/*.whl all-rocm-wheels/ 2>/dev/null || true
done
WHEEL_COUNT=$(find all-rocm-wheels -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l)
echo "Total wheels to upload: $WHEEL_COUNT"
@@ -84,9 +96,9 @@ ls -lh all-rocm-wheels/
# ======== Part 2: Upload wheels to S3 ========
echo ""
echo "Uploading wheels to $S3_COMMIT_PREFIX"
echo "Uploading wheels to $S3_WHEEL_PREFIX"
for wheel in all-rocm-wheels/*.whl; do
aws s3 cp "$wheel" "$S3_COMMIT_PREFIX"
aws s3 cp "$wheel" "$S3_WHEEL_PREFIX"
done
# ======== Part 3: Generate and upload indices ========
@@ -95,28 +107,63 @@ done
echo ""
echo "Generating indices..."
obj_json="rocm-objects.json"
aws s3api list-objects-v2 --bucket "$BUCKET" --prefix "$ROCM_SUBPATH/" --delimiter / --output json > "$obj_json"
aws s3api list-objects-v2 --bucket "$BUCKET" --prefix "$WHEEL_SUBPATH/" --delimiter / --output json > "$obj_json"
mkdir -p "$INDICES_OUTPUT_DIR"
# Use the existing generate-nightly-index.py
# HACK: Replace regex module with stdlib re (same as CUDA script)
sed -i 's/import regex as re/import re/g' .buildkite/scripts/generate-nightly-index.py
sed -i 's/import regex as re/import re/g' tools/vllm-rocm/therock_wheels.py
sed -i 's/import pybase64 as base64/import base64/g' tools/vllm-rocm/therock_wheels.py
INDEX_ARGS=(--version "$ROCM_SUBPATH" --wheel-dir "${WHEEL_SUBPATH#rocm/}")
# TheRock's SDK, device kernels and torchvision/torchaudio are linked, not hosted
if [[ "$ROCM_STACK_THEROCK" == "1" ]]; then
$PYTHON tools/vllm-rocm/therock_wheels.py external-links \
--dockerfile "$ROCM_STACK_BASE_DOCKERFILE" \
--torch-wheel "$(ls "$ROCM_STACK_BASE_WHEELS_DIR"/torch-*.whl)" > therock-external-links.txt
cat therock-external-links.txt
INDEX_ARGS+=(--external-links therock-external-links.txt)
fi
$PYTHON .buildkite/scripts/generate-nightly-index.py \
--version "$ROCM_SUBPATH" \
"${INDEX_ARGS[@]}" \
--current-objects "$obj_json" \
--output-dir "$INDICES_OUTPUT_DIR" \
--comment "ROCm commit $BUILDKITE_COMMIT"
# Upload indices to commit directory
echo "Uploading indices to $S3_COMMIT_PREFIX"
aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "$S3_COMMIT_PREFIX"
# Upload this build's variant indices under an S3 prefix (e.g. rocm/nightly), then
# rebuild that prefix's top-level project list from every variant now present.
publish_indices() {
local prefix="$1"
local listing variants merged
echo "Uploading indices to s3://$BUCKET/$prefix/"
aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/$prefix/" --exclude "index.html"
listing=$(aws s3api list-objects-v2 --bucket "$BUCKET" --prefix "$prefix/" \
--delimiter / --query 'CommonPrefixes[].Prefix' --output text)
variants=$(tr '\t' '\n' <<< "$listing" | sed -n "s|^$prefix/\(rocm[0-9]*\)/$|\1|p" | sort -u)
merged=$(mktemp)
{
echo '<!DOCTYPE html>'
echo '<html>'
echo ' <meta name="pypi:repository-version" content="1.0">'
echo ' <body>'
for variant in $variants; do
echo " <a href=\"$variant/\">$variant/</a><br/>"
done
echo ' </body>'
echo '</html>'
} > "$merged"
echo "Variants under $prefix/: $(echo "$variants" | tr '\n' ' ')"
aws s3 cp "$merged" "s3://$BUCKET/$prefix/index.html"
rm -f "$merged"
}
publish_indices "$ROCM_SUBPATH"
# Only scheduled nightly builds should update the moving nightly index.
if [[ "${NIGHTLY:-0}" == "1" ]]; then
echo "Updating rocm/nightly/ index..."
aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/rocm/nightly/"
publish_indices "rocm/nightly"
fi
# Extract version from vLLM wheel and update version-specific index
@@ -129,8 +176,7 @@ if [ -n "$VLLM_WHEEL" ]; then
echo "Pure version: $PURE_VERSION"
if [[ "$VERSION" != *"dev"* ]]; then
echo "Updating rocm/$PURE_VERSION/ index..."
aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/rocm/$PURE_VERSION/"
publish_indices "rocm/$PURE_VERSION"
fi
fi
@@ -142,14 +188,14 @@ echo "ROCm Wheel Upload Complete!"
echo "========================================"
echo ""
echo "Wheels available at:"
echo " s3://$BUCKET/$ROCM_SUBPATH/"
echo " s3://$BUCKET/$WHEEL_SUBPATH/"
echo ""
echo "Install command (by commit):"
echo " pip install vllm --extra-index-url https://${BUCKET}.s3.amazonaws.com/$ROCM_SUBPATH/"
echo " pip install vllm --extra-index-url https://${BUCKET}.s3.amazonaws.com/$ROCM_SUBPATH/$ROCM_STACK_VARIANT/"
echo ""
if [[ "${NIGHTLY:-0}" == "1" ]]; then
echo "Install command (nightly):"
echo " pip install vllm --extra-index-url https://${BUCKET}.s3.amazonaws.com/rocm/nightly/"
echo " pip install vllm --extra-index-url https://${BUCKET}.s3.amazonaws.com/rocm/nightly/$ROCM_STACK_VARIANT/"
fi
echo ""
echo "Wheel count: $WHEEL_COUNT"