[ROCm][CI] Make ROCm 10.0 the default image; keep ROCm 7.2 as -rocm72 (#58761)

Signed-off-by: Rohan Potdar <rohan.potdar@amd.com>
Signed-off-by: Rohan138 <rohanpotdar138@gmail.com>
Signed-off-by: Micah Williamson <micah.williamson@amd.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Micah Williamson <micah.williamson@amd.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>

.buildkite/release-pipeline.yaml

@@ -703,385 +703,6 @@ steps:
           DOCKER_BUILDKIT: "1"
           DOCKERHUB_USERNAME: "vllmbot"
 
-  # =============================================================================
-  # ROCm Release Pipeline (x86_64 only)
-  # =============================================================================
-  #
-  # vLLM version is determined by the Buildkite checkout (like CUDA pipeline).
-  # To build a specific version, trigger the build from that branch/tag.
-  #
-  # Environment variables for ROCm builds (set via Buildkite UI or schedule):
-  #
-  # Note: ROCm version is determined by BASE_IMAGE in docker/Dockerfile.rocm_base
-  #
-  # =============================================================================
-
-  - group: "Build ROCm Wheel / Image "
-    key: "build-rocm-wheel-image"
-    depends_on: ~
-    steps:
-      # ROCm Job 1: Build ROCm Base Wheels (with S3 caching)
-      - label: ":rocm: Build ROCm Base Image & Wheels"
-        id: build-rocm-base-wheels
-        depends_on: ~
-        agents:
-          queue: cpu_queue_release
-        commands:
-          - |
-            set -euo pipefail
-
-            # Generate cache key
-            CACHE_KEY=$$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
-            ECR_CACHE_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:$${CACHE_KEY}-rocm-base"
-
-            echo "========================================"
-            echo "ROCm Base Build Configuration"
-            echo "========================================"
-            echo "  CACHE_KEY: $${CACHE_KEY}"
-            echo "  ECR_CACHE_TAG: $${ECR_CACHE_TAG}"
-            echo "========================================"
-            
-            # Login to ECR
-            aws ecr-public get-login-password --region us-east-1 | \
-              docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
-            
-            IMAGE_EXISTS=false
-            WHEELS_EXIST=false
-            
-            # Check ECR for Docker image
-
-            if docker manifest inspect "$${ECR_CACHE_TAG}" > /dev/null 2>&1; then
-              IMAGE_EXISTS=true
-              echo "ECR image cache HIT"
-            fi
-            
-            # Check S3 for wheels
-            WHEEL_CACHE_STATUS=$(.buildkite/scripts/cache-rocm-base-wheels.sh check)
-            if [ "$${WHEEL_CACHE_STATUS}" = "hit" ]; then
-              WHEELS_EXIST=true
-              echo "S3 wheels cache HIT"
-            fi
-
-            
-            # Scenario 1: Both cached (best case)
-            if [ "$${IMAGE_EXISTS}" = "true" ] && [ "$${WHEELS_EXIST}" = "true" ]; then
-              echo ""
-              echo "FULL CACHE HIT - Reusing both image and wheels"
-              echo ""
-
-              # Download wheels
-              .buildkite/scripts/cache-rocm-base-wheels.sh download
-              
-              # Save ECR tag for downstream jobs
-              buildkite-agent meta-data set "rocm-base-image-tag" "$${ECR_CACHE_TAG}"
-              
-            # Scenario 2: Full rebuild needed
-            else
-              echo ""
-              echo " CACHE MISS - Building from scratch..."
-              echo ""
-              
-              # Build full base image and push to ECR
-              DOCKER_BUILDKIT=1 docker buildx build \
-                --file docker/Dockerfile.rocm_base \
-                --tag "$${ECR_CACHE_TAG}" \
-                --build-arg USE_SCCACHE=1 \
-                --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
-                --build-arg SCCACHE_REGION_NAME=us-west-2 \
-                --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
-                --push \
-                .
-              
-              # Build wheel extraction stage
-              DOCKER_BUILDKIT=1 docker buildx build \
-                --file docker/Dockerfile.rocm_base \
-                --tag rocm-base-debs:$${BUILDKITE_BUILD_NUMBER} \
-                --target debs_wheel_release \
-                --build-arg USE_SCCACHE=1 \
-                --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
-                --build-arg SCCACHE_REGION_NAME=us-west-2 \
-                --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
-                --load \
-                .
-              
-              # Extract and upload wheels
-              mkdir -p artifacts/rocm-base-wheels
-              cid=$(docker create rocm-base-debs:$${BUILDKITE_BUILD_NUMBER})
-              docker cp $${cid}:/app/debs/. artifacts/rocm-base-wheels/
-              docker rm $${cid}
-              
-              .buildkite/scripts/cache-rocm-base-wheels.sh upload
-
-              # Cache base docker image to ECR
-              docker push "$${ECR_CACHE_TAG}"
-              
-              buildkite-agent meta-data set "rocm-base-image-tag" "$${ECR_CACHE_TAG}"
-              
-              echo ""
-              echo " Build complete - Image and wheels cached"
-            fi
-
-        artifact_paths:
-          - "artifacts/rocm-base-wheels/*.whl"
-        env:
-          DOCKER_BUILDKIT: "1"
-          S3_BUCKET: "vllm-wheels"
-
-      # ROCm Job 2: Build vLLM ROCm Wheel
-      - label: ":python: Build vLLM ROCm Wheel - x86_64"
-        id: build-rocm-vllm-wheel
-        depends_on:
-          - step: build-rocm-base-wheels
-            allow_failure: false
-        agents:
-          queue: cpu_queue_release
-        timeout_in_minutes: 180
-        commands:
-          # Download artifacts and prepare Docker image
-          - |
-            set -euo pipefail
-
-            # Ensure git tags are up-to-date (Buildkite's default fetch doesn't update tags)
-            # This fixes version detection when tags are moved/force-pushed
-            echo "Fetching latest tags from origin..."
-            git fetch --tags --force origin
-            
-            # Log tag information for debugging version detection
-            echo "========================================"
-            echo "Git Tag Verification"
-            echo "========================================"
-            echo "Current HEAD: $(git rev-parse HEAD)"
-            echo "git describe --tags: $(git describe --tags 2>/dev/null || echo 'No tags found')"
-            echo ""
-            echo "Recent tags (pointing to commits near HEAD):"
-            git tag -l --sort=-creatordate | head -5
-            echo "setuptools_scm version detection:"
-            pip install -q setuptools_scm 2>/dev/null || true
-            python3 -c "import setuptools_scm; print('  Detected version:', setuptools_scm.get_version())" 2>/dev/null || echo "  (setuptools_scm not available in this environment)"
-            echo "========================================"
-
-            # Download wheel artifacts from current build
-            echo "Downloading wheel artifacts from current build"
-            buildkite-agent artifact download "artifacts/rocm-base-wheels/*.whl" .
-
-            # Get ECR image tag from metadata (set by build-rocm-base-wheels)
-            ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rocm-base-image-tag 2>/dev/null || echo '')"
-            if [ -z "$${ECR_IMAGE_TAG}" ]; then
-              echo "ERROR: rocm-base-image-tag metadata not found"
-              echo "This should have been set by the build-rocm-base-wheels job"
-              exit 1
-            fi
-            
-            echo "Pulling base Docker image from ECR: $${ECR_IMAGE_TAG}"
-            
-            # Login to ECR
-            aws ecr-public get-login-password --region us-east-1 | \
-              docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
-            
-            # Pull base Docker image from ECR
-            docker pull "$${ECR_IMAGE_TAG}"
-            
-            echo "Loaded base image: $${ECR_IMAGE_TAG}"
-            
-            # Prepare base wheels for Docker build context
-            mkdir -p docker/context/base-wheels
-            touch docker/context/base-wheels/.keep
-            cp artifacts/rocm-base-wheels/*.whl docker/context/base-wheels/
-            echo "Base wheels for vLLM build:"
-            ls -lh docker/context/base-wheels/
-
-            echo "========================================"
-            echo "Building vLLM wheel with:"
-            echo "  BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}"
-            echo "  BUILDKITE_BRANCH: $${BUILDKITE_BRANCH}"
-            echo "  BASE_IMAGE: $${ECR_IMAGE_TAG}"
-            echo "========================================"
-
-            # Build vLLM wheel using local checkout (REMOTE_VLLM=0)
-            DOCKER_BUILDKIT=1 docker build \
-              --file docker/Dockerfile.rocm \
-              --target export_vllm_wheel_release \
-              --output type=local,dest=rocm-dist \
-              --build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \
-              --build-arg REMOTE_VLLM=0 \
-              --build-arg GIT_REPO_CHECK=1 \
-              --build-arg USE_SCCACHE=1 \
-              --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
-              --build-arg SCCACHE_REGION_NAME=us-west-2 \
-              --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
-              .
-            echo "Built vLLM wheel:"
-            ls -lh rocm-dist/*.whl
-            # Copy wheel to artifacts directory
-            mkdir -p artifacts/rocm-vllm-wheel
-            cp rocm-dist/*.whl artifacts/rocm-vllm-wheel/
-            echo "Final vLLM wheel:"
-            ls -lh artifacts/rocm-vllm-wheel/
-        artifact_paths:
-          - "artifacts/rocm-vllm-wheel/*.whl"
-        env:
-          DOCKER_BUILDKIT: "1"
-          S3_BUCKET: "vllm-wheels"
-
-      # ROCm Job 3: Upload Wheels to S3
-      - label: ":s3: Upload ROCm Wheels to S3"
-        id: upload-rocm-wheels
-        depends_on:
-          - step: build-rocm-vllm-wheel
-            allow_failure: false
-        agents:
-          queue: cpu_queue_release
-        timeout_in_minutes: 60
-        commands:
-          # Download all wheel artifacts and run upload
-          - |
-            set -euo pipefail
-
-            # Download artifacts from current build
-            echo "Downloading artifacts from current build"
-            buildkite-agent artifact download "artifacts/rocm-base-wheels/*.whl" .
-            buildkite-agent artifact download "artifacts/rocm-vllm-wheel/*.whl" .
-
-            # # Run upload script
-            bash .buildkite/scripts/upload-rocm-wheels.sh
-        env:
-          DOCKER_BUILDKIT: "1"
-          S3_BUCKET: "vllm-wheels"
-
-      # ROCm Job 4: Annotate ROCm Wheel Release
-      - label: ":memo: Annotate ROCm wheel release"
-        id: annotate-rocm-release
-        depends_on:
-          - upload-rocm-wheels
-        agents:
-          queue: cpu_queue_release
-        commands:
-          - "bash .buildkite/scripts/annotate-rocm-release.sh"
-        env:
-          S3_BUCKET: "vllm-wheels"
-
-      # ROCm Job 5: Generate Root Index for ROCm Wheels (for release only)
-      # This is the job to create https://wheels.vllm.ai/rocm/ index allowing
-      # users to install with `uv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm/`
-      - block: "Generate Root Index for ROCm Wheels for Release"
-        key: block-generate-root-index-rocm-wheels
-        depends_on: upload-rocm-wheels
-
-      - label: ":package: Generate Root Index for ROCm Wheels for Release"
-        depends_on: block-generate-root-index-rocm-wheels
-        id: generate-root-index-rocm-wheels
-        agents:
-          queue: cpu_queue_release
-        commands:
-          - "bash tools/vllm-rocm/generate-rocm-wheels-root-index.sh"
-        env:
-          S3_BUCKET: "vllm-wheels"
-          VARIANT: "rocm723"
-
-      # ROCm Job 6: Build ROCm Release Docker Image
-      - label: ":docker: Build release image - x86_64 - ROCm"
-        id: build-rocm-release-image
-        depends_on:
-          - step: block-build-release-images
-            allow_failure: true
-          - step: build-rocm-base-wheels
-            allow_failure: false
-        agents:
-          queue: cpu_queue_release
-        timeout_in_minutes: 60
-        commands:
-          - |
-            set -euo pipefail
-            
-            # Login to ECR
-            aws ecr-public get-login-password --region us-east-1 | \
-              docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
-            
-            # Get ECR image tag from metadata (set by build-rocm-base-wheels)
-            ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rocm-base-image-tag 2>/dev/null || echo '')"
-            if [ -z "$${ECR_IMAGE_TAG}" ]; then
-              echo "ERROR: rocm-base-image-tag metadata not found"
-              echo "This should have been set by the build-rocm-base-wheels job"
-              exit 1
-            fi
-            
-            echo "Pulling base Docker image from ECR: $${ECR_IMAGE_TAG}"
-            
-            # Pull base Docker image from ECR
-            docker pull "$${ECR_IMAGE_TAG}"
-            
-            echo "Loaded base image: $${ECR_IMAGE_TAG}"
-            
-            # Pass the base image ECR tag to downstream steps (nightly publish)
-            buildkite-agent meta-data set "rocm-base-ecr-tag" "$${ECR_IMAGE_TAG}"
-            
-            echo "========================================"
-            echo "Building vLLM ROCm release image with:"
-            echo "  BASE_IMAGE: $${ECR_IMAGE_TAG}"
-            echo "  BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}"
-            echo "========================================"
-            
-            # Build vLLM ROCm release image using cached base
-            DOCKER_BUILDKIT=1 docker build \
-              --build-arg max_jobs=16 \
-              --build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \
-              --build-arg USE_SCCACHE=1 \
-              --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
-              --build-arg SCCACHE_REGION_NAME=us-west-2 \
-              --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
-              --build-arg INSTALL_LMCACHE=true \
-              --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm \
-              --target vllm-openai \
-              --progress plain \
-              -f docker/Dockerfile.rocm .
-            
-            # Push to ECR
-            docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm
-
-            echo ""
-            echo " Successfully built and pushed ROCm release image"
-            echo "   Image: public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm"
-            echo ""
-        env:
-          DOCKER_BUILDKIT: "1"
-          S3_BUCKET: "vllm-wheels"
-
-      - label: "Publish nightly XPU image to DockerHub"
-        depends_on:
-          - create-manifest-xpu
-        if: build.env("NIGHTLY") == "1"
-        agents:
-          queue: small_cpu_queue_release
-        commands:
-          - "bash .buildkite/scripts/xpu/push-nightly-builds-xpu.sh"
-          - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly- vllm/vllm-openai-xpu"
-        plugins:
-          - docker-login#v3.0.0:
-              username: vllmbot
-              password-env: DOCKERHUB_TOKEN
-        env:
-          DOCKER_BUILDKIT: "1"
-          DOCKERHUB_USERNAME: "vllmbot"
-
-      - label: "Publish nightly ROCm image to DockerHub"
-        depends_on:
-          - build-rocm-release-image
-        if: build.env("NIGHTLY") == "1"
-        agents:
-          queue: small_cpu_queue_release
-        commands:
-          - "bash .buildkite/scripts/push-nightly-builds-rocm.sh"
-          # Clean up old nightly builds (keep only last 14)
-          - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly- vllm/vllm-openai-rocm"
-          - "bash .buildkite/scripts/cleanup-nightly-builds.sh base-nightly- vllm/vllm-openai-rocm"
-        plugins:
-          - docker-login#v3.0.0:
-              username: vllmbot
-              password-env: DOCKERHUB_TOKEN
-        env:
-          DOCKER_BUILDKIT: "1"
-          DOCKERHUB_USERNAME: "vllmbot"
-
   - trigger: "perf-eval"
     label: "Trigger perf-eval (CUDA)"
     key: trigger-perf-eval-cuda
@@ -1103,7 +724,7 @@ steps:
     key: trigger-perf-eval-rocm
     if: build.env("NIGHTLY") == "1" || build.env("RUN_PERF_EVAL") == "1"
     depends_on:
-      - build-rocm-release-image
+      - build-rocm100-release-image
     async: true
     soft_fail: true
     build:
@@ -1112,138 +733,250 @@ steps:
       env:
         NIGHTLY: 1 # run all perf-eval workloads
         VLLM_COMMIT: "$BUILDKITE_COMMIT"
-        VLLM_IMAGE_ROCM: "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-rocm"
+        VLLM_IMAGE_ROCM: "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-rocm100"
 
   # =============================================================================
-  # ROCk Release Pipeline (x86_64 only)
+  # ROCm Release Pipelines (x86_64 only)
   # =============================================================================
   #
-  # Same shape as the ROCm pipeline above (Jobs 1 and 6 only), but built from
+  # One group per ROCm stack, identified by its base Dockerfile (ROCM_BASE_DOCKERFILE).
-  # docker/Dockerfile.rock_base and docker/Dockerfile.rock, which install the
+  # The scripts derive the ROCm version, wheel variant, cache keys and image tags from
-  # ROCm SDK from TheRock wheels instead of the ROCm apt repos.
+  # it (.buildkite/scripts/rocm/stack.sh), so both groups share one set of step
+  # definitions, anchored on the ROCm 10.0 group.
   #
-  # Note: ROCm SDK version is determined by ROCM_SDK_VERSION in
+  # - ROCm 10.0 (TheRock, docker/Dockerfile.rocm{,_base}) is the default: it owns the
-  # docker/Dockerfile.rock_base
+  #   :nightly / :latest images and the root wheel index.
+  # - ROCm 7.2 (docker/Dockerfile.rocm_72{,_base}) is kept for a transition period as
+  #   rocm72 images and wheels.
   #
+  # vLLM version is determined by the Buildkite checkout (like the CUDA pipeline).
   # =============================================================================
 
-  - group: "Build ROCk Image"
+  - group: "Build ROCm 10.0 Wheel / Image"
-    key: "build-rock-image"
+    key: "build-rocm100"
     depends_on: ~
     steps:
-      # ROCk Job 1: Build ROCk Base Image (with ECR caching)
+      - label: ":rocm: Build ROCm 10.0 Base Image & Wheels"
-      - label: ":rocm: Build ROCk Base Image"
+        id: build-rocm100-base
-        id: build-rock-base-image
         depends_on: ~
-        agents:
+        agents: &rocm_build_agents
           queue: cpu_queue_release
         timeout_in_minutes: 360
-        commands:
+        env: &rocm100_env
+          ROCM_BASE_DOCKERFILE: "docker/Dockerfile.rocm_base"
+          ROCM_DEFAULT_ARG: "--default"
+          DOCKER_BUILDKIT: "1"
+          S3_BUCKET: "vllm-wheels"
+          DOCKERHUB_USERNAME: "vllmbot"
+        commands: &rocm_base_commands
           - |
             set -euo pipefail
-
+            source .buildkite/scripts/rocm/stack.sh "$${ROCM_BASE_DOCKERFILE}"
-            # The SDK version, torch wheels and component branches all live in
+            SCCACHE_ARGS=(--build-arg USE_SCCACHE=1 --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache
-            # the Dockerfile, so its hash is the cache key.
+              --build-arg SCCACHE_REGION_NAME=us-west-2 --build-arg SCCACHE_S3_NO_CREDENTIALS=0)
-            CACHE_KEY=$$(sha256sum docker/Dockerfile.rock_base | cut -c1-16)
-            ECR_CACHE_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:$${CACHE_KEY}-rock-base"
-
-            echo "========================================"
-            echo "ROCk Base Build Configuration"
-            echo "========================================"
-            echo "  CACHE_KEY: $${CACHE_KEY}"
-            echo "  ECR_CACHE_TAG: $${ECR_CACHE_TAG}"
-            echo "========================================"
-
-            # Login to ECR
             aws ecr-public get-login-password --region us-east-1 | \
               docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
 
-            if docker manifest inspect "$${ECR_CACHE_TAG}" > /dev/null 2>&1; then
+            # Base image, cached in ECR by the base Dockerfile's hash
-              echo "ECR image cache HIT"
+            if docker manifest inspect "$${ROCM_STACK_ECR_BASE}" > /dev/null 2>&1; then
+              echo "ECR image cache HIT: $${ROCM_STACK_ECR_BASE}"
             else
-              echo "CACHE MISS - Building from scratch..."
+              docker buildx build --file "$${ROCM_BASE_DOCKERFILE}" --tag "$${ROCM_STACK_ECR_BASE}" \
-
+                "$${SCCACHE_ARGS[@]}" --push .
-              DOCKER_BUILDKIT=1 docker buildx build \
-                --file docker/Dockerfile.rock_base \
-                --tag "$${ECR_CACHE_TAG}" \
-                --build-arg USE_SCCACHE=1 \
-                --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
-                --build-arg SCCACHE_REGION_NAME=us-west-2 \
-                --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
-                --push \
-                .
             fi
 
-            # Save ECR tag for downstream jobs
+            # Base wheels, cached in S3 (see cache-rocm-base-wheels.sh)
-            buildkite-agent meta-data set "rock-base-image-tag" "$${ECR_CACHE_TAG}"
+            CACHE=.buildkite/scripts/cache-rocm-base-wheels.sh
-        env:
+            if [ "$$($${CACHE} "$${ROCM_BASE_DOCKERFILE}" check)" = "hit" ]; then
-          DOCKER_BUILDKIT: "1"
+              $${CACHE} "$${ROCM_BASE_DOCKERFILE}" download
+            else
+              DEBS="rocm-base-debs:$${BUILDKITE_BUILD_NUMBER}-$${ROCM_STACK_VARIANT}"
+              docker buildx build --file "$${ROCM_BASE_DOCKERFILE}" --tag "$${DEBS}" \
+                --target debs_wheel_release "$${SCCACHE_ARGS[@]}" --load .
+              mkdir -p "$${ROCM_STACK_BASE_WHEELS_DIR}"
+              cid=$$(docker create "$${DEBS}")
+              docker cp "$${cid}":/app/debs/. "$${ROCM_STACK_BASE_WHEELS_DIR}/"
+              docker rm "$${cid}"
+              $${CACHE} "$${ROCM_BASE_DOCKERFILE}" upload
+            fi
+            buildkite-agent artifact upload "$${ROCM_STACK_BASE_WHEELS_DIR}/*.whl"
 
-      # ROCk Job 2: Build ROCk Release Docker Image
+      - label: ":python: Build vLLM ROCm 10.0 Wheel - x86_64"
-      - label: ":docker: Build release image - x86_64 - ROCk"
+        id: build-rocm100-vllm-wheel
-        id: build-rock-release-image
+        depends_on: build-rocm100-base
+        agents: *rocm_build_agents
+        timeout_in_minutes: 180
+        env: *rocm100_env
+        commands: &rocm_vllm_wheel_commands
+          - |
+            set -euo pipefail
+            source .buildkite/scripts/rocm/stack.sh "$${ROCM_BASE_DOCKERFILE}"
+            # Buildkite's default fetch doesn't update moved tags; version detection needs them
+            git fetch --tags --force origin
+            buildkite-agent artifact download "$${ROCM_STACK_BASE_WHEELS_DIR}/*.whl" .
+            aws ecr-public get-login-password --region us-east-1 | \
+              docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
+            docker pull "$${ROCM_STACK_ECR_BASE}"
+            mkdir -p docker/context/base-wheels
+            touch docker/context/base-wheels/.keep
+            cp "$${ROCM_STACK_BASE_WHEELS_DIR}"/*.whl docker/context/base-wheels/
+            docker build --file "$${ROCM_STACK_DOCKERFILE}" --target export_vllm_wheel_release \
+              --output type=local,dest=rocm-dist \
+              --build-arg BASE_IMAGE="$${ROCM_STACK_ECR_BASE}" --build-arg REMOTE_VLLM=0 \
+              --build-arg GIT_REPO_CHECK=1 --build-arg USE_SCCACHE=1 \
+              --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
+              --build-arg SCCACHE_REGION_NAME=us-west-2 --build-arg SCCACHE_S3_NO_CREDENTIALS=0 .
+            mkdir -p "$${ROCM_STACK_VLLM_WHEEL_DIR}"
+            cp rocm-dist/*.whl "$${ROCM_STACK_VLLM_WHEEL_DIR}/"
+            buildkite-agent artifact upload "$${ROCM_STACK_VLLM_WHEEL_DIR}/*.whl"
+
+      - label: ":s3: Upload ROCm 10.0 Wheels to S3"
+        id: upload-rocm100-wheels
+        depends_on: build-rocm100-vllm-wheel
+        agents: *rocm_build_agents
+        timeout_in_minutes: 60
+        env: *rocm100_env
+        commands: &rocm_upload_commands
+          - |
+            set -euo pipefail
+            source .buildkite/scripts/rocm/stack.sh "$${ROCM_BASE_DOCKERFILE}"
+            buildkite-agent artifact download "$${ROCM_STACK_BASE_WHEELS_DIR}/*.whl" .
+            buildkite-agent artifact download "$${ROCM_STACK_VLLM_WHEEL_DIR}/*.whl" .
+            bash .buildkite/scripts/upload-rocm-wheels.sh "$${ROCM_BASE_DOCKERFILE}"
+
+      - label: ":memo: Annotate ROCm 10.0 wheel release"
+        id: annotate-rocm100-release
+        depends_on: upload-rocm100-wheels
+        agents: *rocm_build_agents
+        env: *rocm100_env
+        commands: &rocm_annotate_commands
+          - bash .buildkite/scripts/annotate-rocm-release.sh "$${ROCM_BASE_DOCKERFILE}" $${ROCM_DEFAULT_ARG}
+
+      # Generate https://wheels.vllm.ai/rocm/ (the default stack's variant), so users can
+      # `uv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm/` (release only)
+      - block: "Generate Root Index for ROCm Wheels for Release"
+        key: block-generate-root-index-rocm-wheels
+        depends_on:
+          - upload-rocm100-wheels
+          - upload-rocm72-wheels
+
+      - label: ":package: Generate Root Index for ROCm Wheels for Release"
+        depends_on: block-generate-root-index-rocm-wheels
+        id: generate-root-index-rocm-wheels
+        agents: *rocm_build_agents
+        env: *rocm100_env
+        commands:
+          - "bash tools/vllm-rocm/generate-rocm-wheels-root-index.sh"
+
+      - label: ":docker: Build release image - x86_64 - ROCm 10.0"
+        id: build-rocm100-release-image
         depends_on:
           - step: block-build-release-images
             allow_failure: true
-          - step: build-rock-base-image
+          - step: build-rocm100-base
             allow_failure: false
-        agents:
+        agents: *rocm_build_agents
-          queue: cpu_queue_release
         timeout_in_minutes: 180
-        commands:
+        env: *rocm100_env
+        commands: &rocm_release_image_commands
           - |
             set -euo pipefail
-
+            source .buildkite/scripts/rocm/stack.sh "$${ROCM_BASE_DOCKERFILE}"
-            # Login to ECR
             aws ecr-public get-login-password --region us-east-1 | \
               docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
-
+            docker pull "$${ROCM_STACK_ECR_BASE}"
-            ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rock-base-image-tag 2>/dev/null || echo '')"
+            docker build --file "$${ROCM_STACK_DOCKERFILE}" --target vllm-openai \
-            if [ -z "$${ECR_IMAGE_TAG}" ]; then
+              --tag "$${ROCM_STACK_ECR_IMAGE}" --progress plain \
-              echo "ERROR: rock-base-image-tag metadata not found"
+              --build-arg max_jobs=16 --build-arg BASE_IMAGE="$${ROCM_STACK_ECR_BASE}" \
-              echo "This should have been set by the build-rock-base-image job"
+              --build-arg INSTALL_LMCACHE=true --build-arg USE_SCCACHE=1 \
-              exit 1
-            fi
-
-            docker pull "$${ECR_IMAGE_TAG}"
-
-            # Pass the base image ECR tag to downstream steps (nightly publish)
-            buildkite-agent meta-data set "rock-base-ecr-tag" "$${ECR_IMAGE_TAG}"
-
-            echo "========================================"
-            echo "Building vLLM ROCk release image with:"
-            echo "  BASE_IMAGE: $${ECR_IMAGE_TAG}"
-            echo "  BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}"
-            echo "========================================"
-
-            DOCKER_BUILDKIT=1 docker build \
-              --build-arg max_jobs=16 \
-              --build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \
-              --build-arg USE_SCCACHE=1 \
               --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \
-              --build-arg SCCACHE_REGION_NAME=us-west-2 \
+              --build-arg SCCACHE_REGION_NAME=us-west-2 --build-arg SCCACHE_S3_NO_CREDENTIALS=0 .
-              --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \
+            docker push "$${ROCM_STACK_ECR_IMAGE}"
-              --build-arg INSTALL_LMCACHE=true \
-              --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rock \
-              --target vllm-openai \
-              --progress plain \
-              -f docker/Dockerfile.rock .
 
-            docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rock
+      - label: "Publish nightly ROCm 10.0 image to DockerHub"
-        env:
+        id: publish-rocm100-nightly
+        depends_on: build-rocm100-release-image
+        if: build.env("NIGHTLY") == "1"
+        agents: &rocm_publish_agents
+          queue: small_cpu_queue_release
+        env: *rocm100_env
+        commands: &rocm_publish_nightly_commands
+          - bash .buildkite/scripts/push-nightly-builds-rocm.sh "$${ROCM_BASE_DOCKERFILE}" $${ROCM_DEFAULT_ARG}
+        plugins: &rocm_dockerhub_login
+          - docker-login#v3.0.0:
+              username: vllmbot
+              password-env: DOCKERHUB_TOKEN
+
+  - group: "Build ROCm 7.2 Wheel / Image"
+    key: "build-rocm72"
+    depends_on: ~
+    steps:
+      - label: ":rocm: Build ROCm 7.2 Base Image & Wheels"
+        id: build-rocm72-base
+        depends_on: ~
+        agents: *rocm_build_agents
+        timeout_in_minutes: 360
+        env: &rocm72_env
+          ROCM_BASE_DOCKERFILE: "docker/Dockerfile.rocm_72_base"
+          ROCM_DEFAULT_ARG: ""
           DOCKER_BUILDKIT: "1"
+          S3_BUCKET: "vllm-wheels"
+          DOCKERHUB_USERNAME: "vllmbot"
+        commands: *rocm_base_commands
 
-      - label: "Publish nightly ROCk image to DockerHub"
+      - label: ":python: Build vLLM ROCm 7.2 Wheel - x86_64"
+        id: build-rocm72-vllm-wheel
+        depends_on: build-rocm72-base
+        agents: *rocm_build_agents
+        timeout_in_minutes: 180
+        env: *rocm72_env
+        commands: *rocm_vllm_wheel_commands
+
+      - label: ":s3: Upload ROCm 7.2 Wheels to S3"
+        id: upload-rocm72-wheels
+        depends_on: build-rocm72-vllm-wheel
+        agents: *rocm_build_agents
+        timeout_in_minutes: 60
+        env: *rocm72_env
+        commands: *rocm_upload_commands
+
+      - label: ":memo: Annotate ROCm 7.2 wheel release"
+        id: annotate-rocm72-release
+        depends_on: upload-rocm72-wheels
+        agents: *rocm_build_agents
+        env: *rocm72_env
+        commands: *rocm_annotate_commands
+
+      - label: ":docker: Build release image - x86_64 - ROCm 7.2"
+        id: build-rocm72-release-image
         depends_on:
-          - build-rock-release-image
+          - step: block-build-release-images
+            allow_failure: true
+          - step: build-rocm72-base
+            allow_failure: false
+        agents: *rocm_build_agents
+        timeout_in_minutes: 180
+        env: *rocm72_env
+        commands: *rocm_release_image_commands
+
+      - label: "Publish nightly ROCm 7.2 image to DockerHub"
+        id: publish-rocm72-nightly
+        depends_on: build-rocm72-release-image
+        if: build.env("NIGHTLY") == "1"
+        agents: *rocm_publish_agents
+        env: *rocm72_env
+        commands: *rocm_publish_nightly_commands
+        plugins: *rocm_dockerhub_login
+
+      - label: "Publish nightly XPU image to DockerHub"
+        depends_on:
+          - create-manifest-xpu
         if: build.env("NIGHTLY") == "1"
         agents:
           queue: small_cpu_queue_release
         commands:
-          - "bash .buildkite/scripts/push-nightly-builds-rocm.sh rock rocm100"
+          - "bash .buildkite/scripts/xpu/push-nightly-builds-xpu.sh"
-          # Clean up old nightly builds (keep only last 14)
+          - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly- vllm/vllm-openai-xpu"
-          - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly-rocm100- vllm/vllm-openai-rocm"
-          - "bash .buildkite/scripts/cleanup-nightly-builds.sh base-nightly-rocm100- vllm/vllm-openai-rocm"
         plugins:
           - docker-login#v3.0.0:
               username: vllmbot
@@ -1264,7 +997,8 @@ steps:
       - create-multi-arch-manifest-ubuntu2404
       - create-multi-arch-manifest-cuda-12-9-ubuntu2404
       - create-manifest-xpu
-      - build-rocm-release-image
+      - build-rocm100-release-image
+      - build-rocm72-release-image
       - input-release-version
       # Wait for CPU builds if their block steps were unblocked, so publish
       # doesn't race the in-progress CPU build. allow_failure lets publish
@@ -1289,6 +1023,7 @@ steps:
       - "cuda-13-0-ubuntu-24-04"
       - "cuda-12-9-ubuntu-24-04"
       - "rocm"
+      - "rocm72"
       - "xpu"
       - "cpu"
     plugins:

.buildkite/scripts/annotate-rocm-release.sh

@@ -2,15 +2,23 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 #
-# Generate Buildkite annotation for ROCm wheel release
+# Generate Buildkite annotation for a ROCm stack's wheel release.
+#
+# Usage: annotate-rocm-release.sh <base-dockerfile> [--default]
+#   base-dockerfile  e.g. docker/Dockerfile.rocm_base or docker/Dockerfile.rocm_72_base
+#   --default        the stack also owns the plain :latest / :v<ver> Docker Hub tags
 set -ex
 
-# Extract build configuration from Dockerfile.rocm_base (single source of truth)
+# shellcheck source=.buildkite/scripts/rocm/stack.sh
-# Extract ROCm version dynamically from Dockerfile.rocm_base
+source "$(dirname "${BASH_SOURCE[0]}")/rocm/stack.sh" "${1:?Usage: $0 <base-dockerfile> [--default]}"
-# BASE_IMAGE format: rocm/dev-ubuntu-22.04:7.0-complete -> extracts "7.0"
+TAG_VARIANTS=("-${ROCM_STACK_VARIANT}")
-ROCM_VERSION=$(grep -E '^ARG BASE_IMAGE=' docker/Dockerfile.rocm_base | sed -E 's/.*:([0-9]+\.[0-9]+).*/\1/' || echo "unknown")
+if [[ "${2:-}" == "--default" ]]; then
-PYTHON_VERSION=$(grep '^ARG PYTHON_VERSION=' docker/Dockerfile.rocm_base | sed 's/^ARG PYTHON_VERSION=//')
+  TAG_VARIANTS=("" "-${ROCM_STACK_VARIANT}")
-PYTORCH_ROCM_ARCH=$(grep '^ARG PYTORCH_ROCM_ARCH=' docker/Dockerfile.rocm_base | sed 's/^ARG PYTORCH_ROCM_ARCH=//')
+fi
+ROCM_VERSION="$ROCM_STACK_VERSION"
+VARIANT="$ROCM_STACK_VARIANT"
+PYTHON_VERSION=$(sed -nE 's/^ARG PYTHON_VERSION="?([^" ]+)"?.*/\1/p' "$ROCM_STACK_BASE_DOCKERFILE")
+PYTORCH_ROCM_ARCH=$(sed -nE 's/^ARG PYTORCH_ROCM_ARCH="?([^"]+)"?.*/\1/p' "$ROCM_STACK_BASE_DOCKERFILE")
 
 # Get release version, default to 1.0.0.dev for nightly/per-commit builds
 RELEASE_VERSION=$(buildkite-agent meta-data get release-version 2>/dev/null || echo "")
@@ -18,22 +26,32 @@ if [ -z "${RELEASE_VERSION}" ]; then
   RELEASE_VERSION="1.0.0.dev"
 fi
 
-ROCM_BASE_CACHE_KEY=$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
-
 # S3 URLs
 S3_BUCKET="${S3_BUCKET:-vllm-wheels}"
 S3_REGION="${AWS_DEFAULT_REGION:-us-west-2}"
-S3_URL="http://${S3_BUCKET}.s3-website-${S3_REGION}.amazonaws.com"
+S3_HOST="${S3_BUCKET}.s3-website-${S3_REGION}.amazonaws.com"
+S3_URL="http://${S3_HOST}"
+WHEEL_PATH="rocm/${BUILDKITE_COMMIT}/${VARIANT}-wheels/"
 
-# Format ROCm version for path (e.g., "7.1" -> "rocm710")
+DOCKER_CMDS="docker pull ${ROCM_STACK_ECR_BASE}
-ROCM_VERSION_PATH="rocm$(echo "${ROCM_VERSION}" | tr -d '.')"
+docker pull ${ROCM_STACK_ECR_IMAGE}"
-ROCM_PATH="rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}"
+for suffix in "${TAG_VARIANTS[@]}"; do
-buildkite-agent annotate --style 'success' --context 'rocm-release-workflow' << EOF
+  for tag in "latest${suffix}" "v${RELEASE_VERSION}${suffix}"; do
-## ROCm Wheel and Docker Image Releases
+    DOCKER_CMDS+="
+docker tag ${ROCM_STACK_ECR_BASE} vllm/vllm-openai-rocm:${tag}-base
+docker tag ${ROCM_STACK_ECR_IMAGE} vllm/vllm-openai-rocm:${tag}
+docker push vllm/vllm-openai-rocm:${tag}-base
+docker push vllm/vllm-openai-rocm:${tag}"
+  done
+done
+
+buildkite-agent annotate --style 'success' --context "rocm-release-workflow-${VARIANT}" << EOF
+## ROCm ${ROCM_VERSION} Wheel and Docker Image Releases
 ### Build Configuration
 | Setting | Value |
 |---------|-------|
 | **ROCm Version** | ${ROCM_VERSION} |
+| **Index Variant** | ${VARIANT} |
 | **Python Version** | ${PYTHON_VERSION} |
 | **GPU Architectures** | ${PYTORCH_ROCM_ARCH} |
 | **Branch** | \`${BUILDKITE_BRANCH}\` |
@@ -41,49 +59,20 @@ buildkite-agent annotate --style 'success' --context 'rocm-release-workflow' <<
 
 ### :package: Installation
 
-**Install from this build (by commit):**
-
 \`\`\`bash
-pip install vllm --extra-index-url ${S3_URL}/${ROCM_PATH}/ --trusted-host ${S3_BUCKET}.s3-website-${S3_REGION}.amazonaws.com
+# This build (by commit)
-
+pip install vllm --extra-index-url ${S3_URL}/rocm/${BUILDKITE_COMMIT}/${VARIANT}/ --trusted-host ${S3_HOST}
-# Example for ROCm ${ROCM_VERSION}:
+# Nightly (if published)
-pip install vllm --extra-index-url ${S3_URL}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/ --trusted-host ${S3_BUCKET}.s3-website-${S3_REGION}.amazonaws.com
+pip install vllm --extra-index-url ${S3_URL}/rocm/nightly/${VARIANT}/ --trusted-host ${S3_HOST}
-\`\`\`
-
-**Install from nightly (if published):**
-
-\`\`\`bash
-pip install vllm --extra-index-url ${S3_URL}/rocm/nightly/ --trusted-host ${S3_BUCKET}.s3-website-${S3_REGION}.amazonaws.com
 \`\`\`
 
 ### :floppy_disk: Download Wheels Directly
 
 \`\`\`bash
-# List all ROCm wheels
+aws s3 ls s3://${S3_BUCKET}/${WHEEL_PATH}
-aws s3 ls s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/
+aws s3 cp --recursive --exclude "*" --include "*.whl" s3://${S3_BUCKET}/${WHEEL_PATH} .
-# Download specific wheels
-aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/vllm-*.whl .
-aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/torch-*.whl .
-aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/triton-*.whl .
-aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/triton-kernels-*.whl .
-aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/torchvision-*.whl .
-aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/torchaudio-*.whl .
-aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/amdsmi-*.whl .
-aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/amd_aiter-*.whl .
-aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/flash-attn-*.whl .
 \`\`\`
 
-### :gear: Included Packages
-- **vllm**: vLLM with ROCm support
-- **torch**: PyTorch built for ROCm ${ROCM_VERSION}
-- **triton**: Triton
-- **triton-kernels**: Triton kernels
-- **torchvision**: TorchVision for ROCm PyTorch
-- **torchaudio**: Torchaudio for ROCm PyTorch
-- **amdsmi**: AMD SMI Python bindings
-- **amd_aiter**: Aiter for ROCm
-- **flash-attn**: Flash Attention for ROCm
-
 ### :warning: Notes
 - These wheels are built for **ROCm ${ROCM_VERSION}** and will NOT work with CUDA GPUs
 - Supported GPU architectures: ${PYTORCH_ROCM_ARCH}
@@ -91,23 +80,8 @@ aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/flash-
 
 ### :package: Docker Image Release
 
-To download and upload the image:
-
 \`\`\`
-docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm-base
+${DOCKER_CMDS}
-docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm
-
-docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base
-docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base vllm/vllm-openai-rocm:latest-base
-docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base
-docker push vllm/vllm-openai-rocm:latest-base
-docker push vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base
-
-docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}
-docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT} vllm/vllm-openai-rocm:latest
-docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT} vllm/vllm-openai-rocm:v${RELEASE_VERSION}
-docker push vllm/vllm-openai-rocm:latest
-docker push vllm/vllm-openai-rocm:v${RELEASE_VERSION}
 \`\`\`
 
 EOF

.buildkite/scripts/cache-rocm-base-wheels.sh

@@ -5,39 +5,33 @@
 # Cache helper for ROCm base wheels
 #
 # This script manages caching of pre-built ROCm base wheels (torch, triton, etc.)
-# to avoid rebuilding them when Dockerfile.rocm_base hasn't changed.
+# to avoid rebuilding them when the stack's base Dockerfile hasn't changed.
 #
-# Usage:
+# Usage: cache-rocm-base-wheels.sh <base-dockerfile> <command>
-#   cache-rocm-base-wheels.sh check    - Check if cache exists, outputs "hit" or "miss"
+#   base-dockerfile  e.g. docker/Dockerfile.rocm_base or docker/Dockerfile.rocm_72_base
-#   cache-rocm-base-wheels.sh upload   - Upload wheels to cache
+# Commands:
-#   cache-rocm-base-wheels.sh download - Download wheels from cache
+#   check    - Check if cache exists, outputs "hit" or "miss"
-#   cache-rocm-base-wheels.sh key      - Output the cache key
+#   upload   - Upload wheels to cache
+#   download - Download wheels from cache
+#   key      - Output the cache key
+#   path     - Output the full S3 cache path
 #
 # Environment variables:
 #   S3_BUCKET          - S3 bucket name (default: vllm-wheels)
 #
-# Note: ROCm version is determined by BASE_IMAGE in Dockerfile.rocm_base,
+# The cache key hashes the base Dockerfile and the repo files it COPYs in, which
-#       so changes to ROCm version are captured by the Dockerfile hash.
+# capture the ROCm version and every component pin (see rocm/stack.sh).
 
 set -euo pipefail
 
 BUCKET="${S3_BUCKET:-vllm-wheels}"
-DOCKERFILE="docker/Dockerfile.rocm_base"
 CACHE_PREFIX="rocm/cache"
 
-# Generate hash from Dockerfile content + build args
+# shellcheck source=.buildkite/scripts/rocm/stack.sh
-generate_cache_key() {
+source "$(dirname "${BASH_SOURCE[0]}")/rocm/stack.sh" "${1:?Usage: $0 <base-dockerfile> <command>}"
-    # Include Dockerfile content
+shift
-    if [[ ! -f "$DOCKERFILE" ]]; then
+CACHE_KEY="$ROCM_STACK_WHEEL_KEY"
-        echo "ERROR: Dockerfile not found: $DOCKERFILE" >&2
+WHEELS_DIR="$ROCM_STACK_BASE_WHEELS_DIR"
-        exit 1
-    fi
-    local dockerfile_hash=$(sha256sum "$DOCKERFILE" | cut -c1-16)
-
-    echo "${dockerfile_hash}"
-}
-
-CACHE_KEY=$(generate_cache_key)
 CACHE_PATH="s3://${BUCKET}/${CACHE_PREFIX}/${CACHE_KEY}/"
 
 case "${1:-}" in
@@ -67,19 +61,19 @@ case "${1:-}" in
         echo "Cache path: ${CACHE_PATH}"
         echo ""
 
-        if [[ ! -d "artifacts/rocm-base-wheels" ]]; then
+        if [[ ! -d "${WHEELS_DIR}" ]]; then
-            echo "ERROR: artifacts/rocm-base-wheels directory not found" >&2
+            echo "ERROR: ${WHEELS_DIR} directory not found" >&2
             exit 1
         fi
 
-        WHEEL_COUNT=$(find artifacts/rocm-base-wheels -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l)
+        WHEEL_COUNT=$(find "${WHEELS_DIR}" -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l)
         if [[ "$WHEEL_COUNT" -eq 0 ]]; then
-            echo "ERROR: No wheels found in artifacts/rocm-base-wheels/" >&2
+            echo "ERROR: No wheels found in ${WHEELS_DIR}/" >&2
             exit 1
         fi
 
         echo "Uploading $WHEEL_COUNT wheels..."
-        aws s3 cp --recursive artifacts/rocm-base-wheels/ "${CACHE_PATH}"
+        aws s3 cp --recursive "${WHEELS_DIR}/" "${CACHE_PATH}"
 
         echo ""
         echo "Cache upload complete!"
@@ -93,17 +87,17 @@ case "${1:-}" in
         echo "Cache key: ${CACHE_KEY}"
         echo "Cache path: ${CACHE_PATH}"
         echo ""
-        mkdir -p artifacts/rocm-base-wheels
+        mkdir -p "${WHEELS_DIR}"
         
         # Use sync with include/exclude to only download .whl files
-        aws s3 sync "${CACHE_PATH}" artifacts/rocm-base-wheels/ \
+        aws s3 sync "${CACHE_PATH}" "${WHEELS_DIR}/" \
             --exclude "*" \
             --include "*.whl"
         
         echo ""
         echo "Downloaded wheels:"
-        find artifacts/rocm-base-wheels -maxdepth 1 -name '*.whl' -exec ls -lh {} \;
+        find "${WHEELS_DIR}" -maxdepth 1 -name '*.whl' -exec ls -lh {} \;
-        WHEEL_COUNT=$(find artifacts/rocm-base-wheels -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l)
+        WHEEL_COUNT=$(find "${WHEELS_DIR}" -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l)
         echo ""
         echo "Total: $WHEEL_COUNT wheels"
         echo "========================================"
@@ -118,12 +112,12 @@ case "${1:-}" in
         ;;
 
     *)
-        echo "Usage: $0 {check|upload|download|key|path}" >&2
+        echo "Usage: $0 <base-dockerfile> {check|upload|download|key|path}" >&2
         echo "" >&2
         echo "Commands:" >&2
         echo "  check    - Check if cache exists, outputs 'hit' or 'miss'" >&2
-        echo "  upload   - Upload wheels from artifacts/rocm-base-wheels/ to cache" >&2
+        echo "  upload   - Upload wheels from ${WHEELS_DIR}/ to cache" >&2
-        echo "  download - Download wheels from cache to artifacts/rocm-base-wheels/" >&2
+        echo "  download - Download wheels from cache to ${WHEELS_DIR}/" >&2
         echo "  key      - Output the cache key" >&2
         echo "  path     - Output the full S3 cache path" >&2
         exit 1

.buildkite/scripts/ci-bake-rocm.sh

@@ -17,7 +17,7 @@ DEFAULT_REPO_SLUG="vllm-project/vllm"
 DEFAULT_CI_HCL_SOURCE="docker/ci-rocm.hcl"
 DEFAULT_CI_BASE_CONTENT_FILES=".dockerignore requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt tools/install_torchcodec_rocm.sh rust-toolchain.toml tests/vllm_test_utils"
 DEFAULT_CI_BASE_DOCKERFILE="docker/Dockerfile.rocm"
-DEFAULT_CI_BASE_DOCKERFILE_STAGES="base rust_toolchain_input_0 rust-toolchain-input rust-toolchain build_nixl lmcache_source build_lmcache build_rocshmem build_deepep mori_base ci_base"
+DEFAULT_CI_BASE_DOCKERFILE_STAGES="base rust_toolchain_input_0 rust-toolchain-input rust-toolchain build_nixl lmcache_source build_lmcache build_deepep mori_base ci_base"
 DEFAULT_CI_BASE_METADATA_VERSION="3"
 # ROCm CI forces REMOTE_VLLM=0, so content identity covers only the selected
 # local-source stages rather than unreachable remote-fetch alternatives.
@@ -555,9 +555,6 @@ configure_custom_rocm_stages() {
     CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${stages}}"
     ROCM_CSRC_DOCKERFILE_STAGES="${ROCM_CSRC_DOCKERFILE_STAGES:-${stages}}"
     ROCM_RUST_DOCKERFILE_STAGES="${ROCM_RUST_DOCKERFILE_STAGES:-${stages}}"
-    if [[ " ${stages} " != *" build_rocshmem "* ]]; then
-        unset ROCSHMEM_BRANCH ROCSHMEM_CACHE_KEY DEEPEP_CACHE_KEY
-    fi
 }
 
 compose_dependency_cache_key() {
@@ -1724,14 +1721,11 @@ ci_base_metadata_pairs() {
     metadata_pair "vllm.rocm.nixl_commit" "${NIXL_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "NIXL_BRANCH")}"
     metadata_pair "vllm.rocm.ucx_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "UCX_REPO")"
     metadata_pair "vllm.rocm.ucx_commit" "${UCX_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "UCX_BRANCH")}"
-    metadata_pair "vllm.rocm.rocshmem_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "ROCSHMEM_REPO")"
-    metadata_pair "vllm.rocm.rocshmem_commit" "${ROCSHMEM_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "ROCSHMEM_BRANCH")}"
     metadata_pair "vllm.rocm.deepep_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_REPO")"
     metadata_pair "vllm.rocm.deepep_commit" "${DEEPEP_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_BRANCH")}"
     metadata_pair "vllm.rocm.deepep_nic" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_NIC")"
     metadata_pair "vllm.rocm.deepep_rocm_arch" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_ROCM_ARCH")"
     metadata_pair "vllm.rocm.nixl_cache_key" "${NIXL_CACHE_KEY:-}"
-    metadata_pair "vllm.rocm.rocshmem_cache_key" "${ROCSHMEM_CACHE_KEY:-}"
     metadata_pair "vllm.rocm.deepep_cache_key" "${DEEPEP_CACHE_KEY:-}"
 }
 
@@ -2314,7 +2308,7 @@ extract_dependency_pins() {
         return 0
     fi
 
-    for var in NIXL_BRANCH UCX_BRANCH ROCSHMEM_BRANCH DEEPEP_BRANCH; do
+    for var in NIXL_BRANCH UCX_BRANCH DEEPEP_BRANCH; do
         if [[ -n "${!var:-}" ]]; then
             echo "Using provided ${var}: ${!var}"
             continue
@@ -2336,17 +2330,14 @@ compute_dependency_cache_keys() {
     local dockerfile_rocm=""
     local nixl_branch=""
     local ucx_branch=""
-    local rocshmem_branch=""
     local deepep_branch=""
     local nixl_material=""
-    local rocshmem_material=""
     local deepep_material=""
 
     bake_dir=$(dirname "${VLLM_BAKE_FILE}")
     dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}"
     nixl_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "NIXL_BRANCH")
     ucx_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "UCX_BRANCH")
-    rocshmem_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "ROCSHMEM_BRANCH")
     deepep_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "DEEPEP_BRANCH")
 
     if [[ -n "${nixl_branch}" && -n "${ucx_branch}" ]]; then
@@ -2360,22 +2351,11 @@ compute_dependency_cache_keys() {
         echo "NIXL dependency cache key: ${NIXL_CACHE_KEY}"
     fi
 
-    if [[ -n "${rocshmem_branch}" ]]; then
+    if [[ -n "${deepep_branch}" ]]; then
-        rocshmem_material=$(compose_stage_cache_material "${dockerfile_rocm}" "base build_rocshmem")
+        deepep_material=$(compose_stage_cache_material "${dockerfile_rocm}" "base build_deepep")
-        ROCSHMEM_CACHE_KEY=$(
-            compose_dependency_cache_key \
-                "${rocshmem_branch}" \
-                "${rocshmem_material}"
-        )
-        export ROCSHMEM_CACHE_KEY
-        echo "ROCShmem dependency cache key: ${ROCSHMEM_CACHE_KEY}"
-    fi
-
-    if [[ -n "${deepep_branch}" && -n "${rocshmem_branch}" ]]; then
-        deepep_material=$(compose_stage_cache_material "${dockerfile_rocm}" "base build_rocshmem build_deepep")
         DEEPEP_CACHE_KEY=$(
             compose_dependency_cache_key \
-                "${deepep_branch}-rocshmem-${rocshmem_branch}" \
+                "${deepep_branch}" \
                 "${deepep_material}"
         )
         export DEEPEP_CACHE_KEY
@@ -2415,18 +2395,11 @@ dependency_cache_ref_for_target() {
                 printf '%s\n' "${cache_repo}:nixl-rocm-${NIXL_BRANCH}-ucx-${UCX_BRANCH:-}"
             fi
             ;;
-        rocshmem-rocm-ci)
-            if [[ -n "${ROCSHMEM_CACHE_KEY:-}" ]]; then
-                printf '%s\n' "${cache_repo}:rocshmem-rocm-${ROCSHMEM_CACHE_KEY}"
-            elif [[ -n "${ROCSHMEM_BRANCH:-}" ]]; then
-                printf '%s\n' "${cache_repo}:rocshmem-rocm-${ROCSHMEM_BRANCH}"
-            fi
-            ;;
         deepep-rocm-ci)
             if [[ -n "${DEEPEP_CACHE_KEY:-}" ]]; then
                 printf '%s\n' "${cache_repo}:deepep-rocm-${DEEPEP_CACHE_KEY}"
             elif [[ -n "${DEEPEP_BRANCH:-}" ]]; then
-                printf '%s\n' "${cache_repo}:deepep-rocm-${DEEPEP_BRANCH}-rocshmem-${ROCSHMEM_BRANCH:-}"
+                printf '%s\n' "${cache_repo}:deepep-rocm-${DEEPEP_BRANCH}"
             fi
             ;;
     esac
@@ -2444,7 +2417,6 @@ add_dependency_cache_target() {
 resolve_ci_base_dependency_targets() {
     local mode="${ROCM_DEP_CACHE_EXPORT_MODE:-missing}"
     local nixl_ref=""
-    local rocshmem_ref=""
     local deepep_ref=""
 
     [[ "${TARGET}" == "ci-base-rocm-ci-with-deps" ]] || return 0
@@ -2452,7 +2424,7 @@ resolve_ci_base_dependency_targets() {
     case "${mode}" in
         always)
             echo "ROCM_DEP_CACHE_EXPORT_MODE=always; exporting all dependency caches serially"
-            for target in nixl-rocm-ci rocshmem-rocm-ci deepep-rocm-ci; do
+            for target in nixl-rocm-ci deepep-rocm-ci; do
                 if [[ -n "$(dependency_cache_ref_for_target "${target}")" ]]; then
                     add_dependency_cache_target "${target}"
                 fi
@@ -2482,16 +2454,6 @@ resolve_ci_base_dependency_targets() {
         fi
     fi
 
-    if [[ "${mode}" != "always" && -n "${ROCSHMEM_CACHE_KEY:-}" ]]; then
-        rocshmem_ref=$(dependency_cache_ref_for_target "rocshmem-rocm-ci")
-        if dependency_cache_ref_exists "${rocshmem_ref}"; then
-            echo "ROCShmem dependency cache exists: ${rocshmem_ref}"
-        else
-            echo "ROCShmem dependency cache missing; will seed: ${rocshmem_ref}"
-            add_dependency_cache_target "rocshmem-rocm-ci"
-        fi
-    fi
-
     if [[ "${mode}" != "always" && -n "${DEEPEP_CACHE_KEY:-}" ]]; then
         deepep_ref=$(dependency_cache_ref_for_target "deepep-rocm-ci")
         if dependency_cache_ref_exists "${deepep_ref}"; then
@@ -2502,15 +2464,6 @@ resolve_ci_base_dependency_targets() {
         fi
     fi
 
-    # DeepEP inherits from ROCShmem. If ROCShmem is being seeded, seed DeepEP too
-    # so the pair stays consistent for future ci_base rebuilds.
-    if printf '%s\n' "${DEPENDENCY_CACHE_TARGETS[@]}" | grep -qx "rocshmem-rocm-ci" \
-        && ! printf '%s\n' "${DEPENDENCY_CACHE_TARGETS[@]}" | grep -qx "deepep-rocm-ci" \
-        && [[ -n "${DEEPEP_BRANCH:-}" ]]; then
-        echo "ROCShmem cache is missing; also seeding DeepEP cache"
-        add_dependency_cache_target "deepep-rocm-ci"
-    fi
-
     BAKE_TARGETS=("ci-base-rocm-ci")
     if [[ ${#DEPENDENCY_CACHE_TARGETS[@]} -eq 0 ]]; then
         echo "All dependency caches exist; building ci_base without dependency cache exports"

.buildkite/scripts/cleanup-nightly-builds.sh

@@ -5,14 +5,17 @@ set -ex
 # Clean up old nightly builds from DockerHub, keeping only the last 14 builds
 # This script uses DockerHub API to list and delete old tags with specified prefix
 # Tags starting with "nightly-dev" are always excluded from deletion
-# Usage: cleanup-nightly-builds.sh [TAG_PREFIX] [REPO]
+# Usage: cleanup-nightly-builds.sh [TAG_PREFIX] [REPO] [EXCLUDE_PREFIX]
 # Example: cleanup-nightly-builds.sh "nightly-"
 # Example: cleanup-nightly-builds.sh "cu130-nightly-"
 # Example: cleanup-nightly-builds.sh "nightly-" "vllm/vllm-openai-rocm"
+# Example: cleanup-nightly-builds.sh "nightly-" "vllm/vllm-openai-rocm" "nightly-rocm"
+#   (EXCLUDE_PREFIX skips tags owned by another variant sharing TAG_PREFIX)
 
 # Get tag prefix and repo from arguments
 TAG_PREFIX="${1:-nightly-}"
 REPO="${2:-vllm/vllm-openai}"
+EXCLUDE_PREFIX="${3:-}"
 
 echo "Cleaning up tags with prefix: $TAG_PREFIX in repository: $REPO"
 
@@ -57,7 +60,7 @@ get_all_tags() {
         
         # Get both last_updated timestamp and tag name, separated by |
         # Exclude mutable architecture tags and nightly-dev tags from cleanup
-        local tags=$(echo "$response" | jq -r --arg prefix "$TAG_PREFIX" '.results[] | select(.name | startswith($prefix)) | select(.name != ($prefix + "x86_64") and .name != ($prefix + "aarch64")) | select(.name | startswith("nightly-dev") | not) | "\(.last_updated)|\(.name)"')
+        local tags=$(echo "$response" | jq -r --arg prefix "$TAG_PREFIX" --arg exclude "$EXCLUDE_PREFIX" '.results[] | select(.name | startswith($prefix)) | select($exclude == "" or (.name | startswith($exclude) | not)) | select(.name != ($prefix + "x86_64") and .name != ($prefix + "aarch64")) | select(.name | startswith("nightly-dev") | not) | "\(.last_updated)|\(.name)"')
         
         if [ -z "$tags" ]; then
             break

.buildkite/scripts/generate-nightly-index.py

@@ -12,7 +12,7 @@ from dataclasses import asdict, dataclass
 from datetime import datetime
 from pathlib import Path
 from typing import Any
-from urllib.parse import quote
+from urllib.parse import quote, unquote, urlparse
 
 import regex as re
 
@@ -51,6 +51,8 @@ class WheelFileInfo:
     platform_tag: str
     variant: str | None
     filename: str
+    # Absolute URL for files hosted elsewhere (see --external-links)
+    url: str | None = None
 
 
 def parse_from_filename(file: str) -> WheelFileInfo:
@@ -110,6 +112,29 @@ def parse_from_filename(file: str) -> WheelFileInfo:
     )
 
 
+def parse_external_url(url: str) -> WheelFileInfo:
+    """Parse an absolute wheel or sdist URL hosted on another index."""
+    filename = unquote(urlparse(url).path.rsplit("/", 1)[-1])
+    if filename.endswith(".whl"):
+        info = parse_from_filename(filename)
+    else:
+        match = re.match(r"^(?P<name>.+)-(?P<version>[^-]+)\.(tar\.gz|zip)$", filename)
+        if not match:
+            raise ValueError(f"Unsupported external file: {filename}")
+        info = WheelFileInfo(
+            package_name=match.group("name"),
+            version=match.group("version"),
+            build_tag=None,
+            python_tag="source",
+            abi_tag="none",
+            platform_tag="any",
+            variant=None,
+            filename=filename,
+        )
+    info.url = url
+    return info
+
+
 def generate_project_list(subdir_names: list[str], comment: str = "") -> str:
     """Generate project list HTML content linking to each project & variant subdirectory."""
     href_tags = []
@@ -129,14 +154,19 @@ def generate_package_index_and_metadata(
     href_tags = []
     metadata = []
     for file in sorted(wheel_files, key=lambda x: x.filename):
-        relative_path = (
+        if file.url:
-            wheel_base_dir.relative_to(index_base_dir, walk_up=True) / file.filename
+            file_path_quoted = file.url
-        )
+        else:
-        # handle with '+' in URL, and avoid double-encoding '/' and already-encoded '%2B'
+            relative_path = (
-        # NOTE: this is AWS S3 specific behavior!
+                wheel_base_dir.relative_to(index_base_dir, walk_up=True) / file.filename
-        file_path_quoted = quote(relative_path.as_posix(), safe=":%/")
+            )
+            # handle with '+' in URL, and avoid double-encoding '/' and already-encoded '%2B'
+            # NOTE: this is AWS S3 specific behavior!
+            file_path_quoted = quote(relative_path.as_posix(), safe=":%/")
         href_tags.append(f'    <a href="{file_path_quoted}">{file.filename}</a><br/>')
         file_meta = asdict(file)
+        if file_meta["url"] is None:
+            del file_meta["url"]
         file_meta["path"] = file_path_quoted
         metadata.append(file_meta)
     index_str = INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags), comment=comment)
@@ -151,6 +181,7 @@ def generate_index_and_metadata(
     default_variant: str | None = None,
     alias_to_default: str | None = None,
     comment: str = "",
+    external_urls: list[str] | None = None,
 ):
     """Generate index for all wheel files.
 
@@ -160,6 +191,8 @@ def generate_index_and_metadata(
         index_base_dir (Path): Base directory to store index files.
         default_variant (str | None): The default variant name, if any.
         alias_to_default (str | None): Alias variant name for the default variant, if any.
+        external_urls (list[str] | None): Absolute URLs of files hosted on another index,
+            listed in the generated indices without being copied.
         comment (str | None): Optional comment to include in the generated HTML files.
 
     First, parse all wheel files to extract metadata.
@@ -214,6 +247,7 @@ def generate_index_and_metadata(
 
     """
     parsed_files = [parse_from_filename(f) for f in whl_files]
+    parsed_files += [parse_external_url(u) for u in external_urls or []]
 
     if not parsed_files:
         print("No wheel files found, skipping index generation.")
@@ -232,10 +266,14 @@ def generate_index_and_metadata(
             print(f"Detected ROCm variant from vllm: {rocm_variant}")
             break
 
-    # Apply ROCm variant to all wheels without a variant
+    # Apply ROCm variant to all wheels without a variant. Dependencies with their
+    # own ROCm local version (e.g. TheRock's torch-2.12.0+rocm10.0.0) belong to
+    # the same variant rather than one of their own.
     if rocm_variant:
         for file in parsed_files:
-            if file.variant is None:
+            if file.variant is None or (
+                file.variant.startswith("rocm") and file.variant != rocm_variant
+            ):
                 file.variant = rocm_variant
                 print(f"Inherited variant '{rocm_variant}' for {file.filename}")
 
@@ -366,6 +404,13 @@ if __name__ == "__main__":
         default=None,
         help="Alias variant name for the default variant",
     )
+    parser.add_argument(
+        "--external-links",
+        type=str,
+        default=None,
+        help="Optional file with one absolute URL per line for packages hosted on "
+        "another index (listed alongside, but not copied into, this index)",
+    )
     parser.add_argument(
         "--comment",
         type=str,
@@ -463,5 +508,14 @@ if __name__ == "__main__":
         default_variant=None,
         alias_to_default=args.alias_to_default,
         comment=args.comment.strip(),
+        external_urls=(
+            [
+                line.strip()
+                for line in Path(args.external_links).read_text().splitlines()
+                if line.strip() and not line.startswith("#")
+            ]
+            if args.external_links
+            else None
+        ),
     )
     print(f"Successfully generated index and metadata in {output_dir}")

.buildkite/scripts/publish-release-images.sh

@@ -11,9 +11,9 @@ set -euo pipefail
 TARGET="${1:-all}"
 case "${TARGET}" in
   cuda-13-0 | cuda-12-9 | cuda-13-0-ubuntu-24-04 | \
-    cuda-12-9-ubuntu-24-04 | rocm | xpu | cpu | all) ;;
+    cuda-12-9-ubuntu-24-04 | rocm | rocm72 | xpu | cpu | all) ;;
   *)
-    echo "Usage: $0 {cuda-13-0|cuda-12-9|cuda-13-0-ubuntu-24-04|cuda-12-9-ubuntu-24-04|rocm|xpu|cpu|all}"
+    echo "Usage: $0 {cuda-13-0|cuda-12-9|cuda-13-0-ubuntu-24-04|cuda-12-9-ubuntu-24-04|rocm|rocm72|xpu|cpu|all}"
     exit 2
     ;;
 esac
@@ -143,23 +143,35 @@ if target_enabled cuda-12-9-ubuntu-24-04; then
 fi
 
 # ---- ROCm ----
+# Each ROCm stack is identified by its base Dockerfile (see rocm/stack.sh). The
+# default stack also owns the plain :latest / :v<ver> tags.
+
+publish_rocm_stack() {
+  local base_dockerfile="$1" default="$2" suffix
+  (
+    # shellcheck source=.buildkite/scripts/rocm/stack.sh
+    source .buildkite/scripts/rocm/stack.sh "$base_dockerfile"
+    local suffixes=("-${ROCM_STACK_VARIANT}")
+    [[ "$default" == "1" ]] && suffixes=("" "-${ROCM_STACK_VARIANT}")
+    docker pull "$ROCM_STACK_ECR_IMAGE"
+    docker pull "$ROCM_STACK_ECR_BASE"
+    for suffix in "${suffixes[@]}"; do
+      for tag in "latest${suffix}" "v${RELEASE_VERSION}${suffix}"; do
+        docker tag "$ROCM_STACK_ECR_IMAGE" "vllm/vllm-openai-rocm:${tag}"
+        docker tag "$ROCM_STACK_ECR_BASE" "vllm/vllm-openai-rocm:${tag}-base"
+        docker push "vllm/vllm-openai-rocm:${tag}"
+        docker push "vllm/vllm-openai-rocm:${tag}-base"
+      done
+    done
+  )
+}
 
 if target_enabled rocm; then
-  ROCM_BASE_CACHE_KEY=$(.buildkite/scripts/cache-rocm-base-wheels.sh key)
+  publish_rocm_stack docker/Dockerfile.rocm_base 1
-  echo "ROCm base cache key: ${ROCM_BASE_CACHE_KEY}"
+fi
 
-  docker pull "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm"
+if target_enabled rocm72; then
-  docker pull "public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base"
+  publish_rocm_stack docker/Dockerfile.rocm_72_base 0
-
-  docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm" vllm/vllm-openai-rocm:latest
-  docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm" "vllm/vllm-openai-rocm:v${RELEASE_VERSION}"
-  docker push vllm/vllm-openai-rocm:latest
-  docker push "vllm/vllm-openai-rocm:v${RELEASE_VERSION}"
-
-  docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base" vllm/vllm-openai-rocm:latest-base
-  docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base" "vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base"
-  docker push vllm/vllm-openai-rocm:latest-base
-  docker push "vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base"
 fi
 
 # ---- XPU ----

.buildkite/scripts/push-nightly-builds-rocm.sh

@@ -2,42 +2,36 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 #
-# Push a ROCm-family nightly base image and nightly image from ECR to Docker Hub
+# Push a ROCm stack's nightly base image and nightly image from ECR to Docker Hub
-# under vllm/vllm-openai-rocm, as base-nightly[-<variant>], nightly[-<variant>]
+# under vllm/vllm-openai-rocm, as base-nightly-<variant>, nightly-<variant> and their
-# and their -<commit> forms.
+# -<commit> forms, then prune old nightly tags. With --default, also push the plain
-# Run when NIGHTLY=1 after the matching build-*-release-image step has pushed to ECR.
+# base-nightly and nightly tags for the same images.
+# Run when NIGHTLY=1 after the stack's release image has been pushed to ECR.
 #
-# Usage: push-nightly-builds-rocm.sh [ECR_TAG_SUFFIX TAG_VARIANT]
+# Usage: push-nightly-builds-rocm.sh <base-dockerfile> [--default]
-#   Pass both arguments or neither. With none, the default ROCm apt build is
+#   e.g. "docker/Dockerfile.rocm_base --default" (ROCm 10.0, also :nightly)
-#   published as :nightly. The TheRock build passes "rock" (the source ECR tag
+#   and "docker/Dockerfile.rocm_72_base" (ROCm 7.2, :nightly-rocm72).
-#   suffix) and "rocm714" (the Docker Hub tag flavor), giving :nightly-rocm714.
 #
 # Local testing (no push to Docker Hub):
-#   BASE_ECR_IMAGE=<full-base-image-reference> \
+#   BUILDKITE_COMMIT=<commit-with-image-in-ecr> DRY_RUN=1 \
-#     BUILDKITE_COMMIT=<commit-with-image-in-ecr> DRY_RUN=1 \
+#     bash .buildkite/scripts/push-nightly-builds-rocm.sh docker/Dockerfile.rocm_base --default
-#     bash .buildkite/scripts/push-nightly-builds-rocm.sh
 # Requires: AWS CLI configured (for ECR public login), Docker. For full run: Docker Hub login.
 
 set -euxo pipefail
 
 usage() {
-  echo "Usage: $0 [ECR_TAG_SUFFIX TAG_VARIANT]" >&2
+  echo "Usage: $0 <base-dockerfile> [--default]" >&2
 }
 
-DOCKERHUB_REPO="vllm/vllm-openai-rocm"
+PUSH_DEFAULT=0
-
 case "$#" in
-  0)
+  1) ;;
-    ECR_TAG_SUFFIX="rocm"
-    TAG_VARIANT=""
-    ;;
   2)
-    if [[ -z "$1" || -z "$2" ]]; then
+    if [[ "$2" != "--default" ]]; then
       usage
       exit 2
     fi
-    ECR_TAG_SUFFIX="$1"
+    PUSH_DEFAULT=1
-    TAG_VARIANT="$2"
     ;;
   *)
     usage
@@ -49,61 +43,44 @@ esac
 BUILDKITE_COMMIT="${BUILDKITE_COMMIT:?Set BUILDKITE_COMMIT to the commit SHA that has the image in ECR (e.g. from a previous release pipeline run)}"
 DRY_RUN="${DRY_RUN:-0}"
 
-ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-release-repo"
+# shellcheck source=.buildkite/scripts/rocm/stack.sh
+source "$(dirname "${BASH_SOURCE[0]}")/rocm/stack.sh" "$1"
+DOCKERHUB_REPO="vllm/vllm-openai-rocm"
+VARIANT="$ROCM_STACK_VARIANT"
 
-# Get the base image ECR tag (set by the build-*-release-image pipeline step)
+BASE_TAGS=("base-nightly-${VARIANT}" "base-nightly-${VARIANT}-${BUILDKITE_COMMIT}")
-BASE_METADATA_KEY="${ECR_TAG_SUFFIX}-base-ecr-tag"
+TAGS=("nightly-${VARIANT}" "nightly-${VARIANT}-${BUILDKITE_COMMIT}")
-if [[ -n "${BASE_ECR_IMAGE:-}" ]]; then
+if [[ "$PUSH_DEFAULT" == "1" ]]; then
-  if [[ "$DRY_RUN" != "1" ]]; then
+  BASE_TAGS+=("base-nightly" "base-nightly-${BUILDKITE_COMMIT}")
-    echo "ERROR: BASE_ECR_IMAGE may only be used with DRY_RUN=1" >&2
+  TAGS+=("nightly" "nightly-${BUILDKITE_COMMIT}")
-    exit 1
-  fi
-  BASE_ORIG_TAG="$BASE_ECR_IMAGE"
-elif ! BASE_ORIG_TAG="$(buildkite-agent meta-data get "$BASE_METADATA_KEY")"; then
-  echo "ERROR: Failed to read required Buildkite metadata '$BASE_METADATA_KEY'" >&2
-  exit 1
-fi
-if [[ -z "$BASE_ORIG_TAG" ]]; then
-  echo "ERROR: Required Buildkite metadata '$BASE_METADATA_KEY' is empty" >&2
-  echo "Set BASE_ECR_IMAGE to the full ECR base image reference for local testing" >&2
-  exit 1
 fi
 
-ORIG_TAG="${BUILDKITE_COMMIT}-${ECR_TAG_SUFFIX}"
+echo "Pushing base image $ROCM_STACK_ECR_BASE as: ${BASE_TAGS[*]}"
-VARIANT_SUFFIX=""
+echo "Pushing release image $ROCM_STACK_ECR_IMAGE as: ${TAGS[*]}"
-[[ -n "$TAG_VARIANT" ]] && VARIANT_SUFFIX="-$TAG_VARIANT"
-BASE_TAG_NAME="base-nightly${VARIANT_SUFFIX}"
-TAG_NAME="nightly${VARIANT_SUFFIX}"
-BASE_TAG_NAME_COMMIT="base-nightly${VARIANT_SUFFIX}-${BUILDKITE_COMMIT}"
-TAG_NAME_COMMIT="nightly${VARIANT_SUFFIX}-${BUILDKITE_COMMIT}"
 
-echo "Pushing base image from ECR: $BASE_ORIG_TAG"
+# Login to ECR and pull the stack's images
-echo "Pushing release image from ECR tag: $ORIG_TAG to Docker Hub as $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT"
-[[ "$DRY_RUN" == "1" ]] && echo "[DRY_RUN] Skipping push to Docker Hub"
-
-# Login to ECR and pull the images built by build-*-release-image
 aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7
-docker pull "$BASE_ORIG_TAG"
+docker pull "$ROCM_STACK_ECR_BASE"
-docker pull "${ECR_REPO}:${ORIG_TAG}"
+docker pull "$ROCM_STACK_ECR_IMAGE"
 
-# Tag for Docker Hub (base-nightly and base-nightly-<commit>, nightly and nightly-<commit>)
+for tag in "${BASE_TAGS[@]}"; do docker tag "$ROCM_STACK_ECR_BASE" "$DOCKERHUB_REPO:$tag"; done
-docker tag "$BASE_ORIG_TAG" "$DOCKERHUB_REPO":"$BASE_TAG_NAME"
+for tag in "${TAGS[@]}"; do docker tag "$ROCM_STACK_ECR_IMAGE" "$DOCKERHUB_REPO:$tag"; done
-docker tag "$BASE_ORIG_TAG" "$DOCKERHUB_REPO":"$BASE_TAG_NAME_COMMIT"
-docker tag "${ECR_REPO}:${ORIG_TAG}" "$DOCKERHUB_REPO":"$TAG_NAME"
-docker tag "${ECR_REPO}:${ORIG_TAG}" "$DOCKERHUB_REPO":"$TAG_NAME_COMMIT"
 
 if [[ "$DRY_RUN" == "1" ]]; then
-  echo "[DRY_RUN] Would push $DOCKERHUB_REPO:$BASE_TAG_NAME and $DOCKERHUB_REPO:$BASE_TAG_NAME_COMMIT"
-  echo "[DRY_RUN] Would push $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT"
   echo "[DRY_RUN] Local tags created. Exiting without push."
   exit 0
 fi
 
 # Push to Docker Hub (docker-login plugin runs before this step in CI)
-docker push "$DOCKERHUB_REPO":"$BASE_TAG_NAME"
+for tag in "${BASE_TAGS[@]}" "${TAGS[@]}"; do docker push "$DOCKERHUB_REPO:$tag"; done
-docker push "$DOCKERHUB_REPO":"$BASE_TAG_NAME_COMMIT"
+echo "Pushed $DOCKERHUB_REPO: ${BASE_TAGS[*]} ${TAGS[*]}"
-docker push "$DOCKERHUB_REPO":"$TAG_NAME"
-docker push "$DOCKERHUB_REPO":"$TAG_NAME_COMMIT"
 
-echo "Pushed $DOCKERHUB_REPO:$BASE_TAG_NAME and $DOCKERHUB_REPO:$BASE_TAG_NAME_COMMIT"
+# Keep only the last 14 builds of each tag family. The plain nightly- families
-echo "Pushed $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT"
+# exclude the variant ones sharing their prefix; commit SHAs never start with "rocm".
+CLEANUP=.buildkite/scripts/cleanup-nightly-builds.sh
+bash "$CLEANUP" "nightly-${VARIANT}-" "$DOCKERHUB_REPO"
+bash "$CLEANUP" "base-nightly-${VARIANT}-" "$DOCKERHUB_REPO"
+if [[ "$PUSH_DEFAULT" == "1" ]]; then
+  bash "$CLEANUP" "nightly-" "$DOCKERHUB_REPO" "nightly-rocm"
+  bash "$CLEANUP" "base-nightly-" "$DOCKERHUB_REPO" "base-nightly-rocm"
+fi

.buildkite/scripts/rocm/build-config.sh

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 # Shared Dockerfile selection for the normal AMD CI build chain.
 # CI_ROCM_DOCKERFILE_BASE and CI_ROCM_DOCKERFILE accept repository-relative paths.
-# For Rock, set CI_ROCM_DOCKERFILE_BASE=docker/Dockerfile.rock_base and
+# For the legacy ROCm 7.2 stack, set CI_ROCM_DOCKERFILE_BASE=docker/Dockerfile.rocm_72_base
-# CI_ROCM_DOCKERFILE=docker/Dockerfile.rock.
+# and CI_ROCM_DOCKERFILE=docker/Dockerfile.rocm_72.
 # Custom recipes must preserve the CI targets and artifact layout. Additional
 # COPY inputs also need the corresponding *_CONTENT_FILES overrides.
 

.buildkite/scripts/rocm/stack.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+#
+# Derive everything the release pipeline needs about a ROCm stack from its base
+# Dockerfile, so the stack's image is the single input to every ROCm script.
+#
+# Usage: source .buildkite/scripts/rocm/stack.sh <base-dockerfile>
+#   e.g. docker/Dockerfile.rocm_base (ROCm 10.0) or docker/Dockerfile.rocm_72_base
+#
+# Exports:
+#   ROCM_STACK_BASE_DOCKERFILE  the base Dockerfile
+#   ROCM_STACK_DOCKERFILE       the matching vLLM Dockerfile (base name minus _base)
+#   ROCM_STACK_THEROCK          1 if the ROCm SDK comes from TheRock wheels
+#   ROCM_STACK_VERSION          ROCm version, e.g. 10.0.0 or 7.2.3
+#   ROCM_STACK_VARIANT          wheel variant and Docker Hub tag flavor from major.minor,
+#                               same rule as setup.py (rocm100, rocm72)
+#   ROCM_STACK_IMAGE_KEY        hash of the base Dockerfile (base image cache key)
+#   ROCM_STACK_WHEEL_KEY        hash of the base Dockerfile plus files it COPYs from
+#                               the repo (base wheel cache key)
+#   ROCM_STACK_ECR_BASE         ECR tag of the cached base image
+#   ROCM_STACK_ECR_IMAGE        ECR tag of this commit's release image
+#   ROCM_STACK_BASE_WHEELS_DIR  local dir for base wheels (also the artifact path)
+#   ROCM_STACK_VLLM_WHEEL_DIR   local dir for the vLLM wheel (also the artifact path)
+
+rocm_stack_init() {
+    local base="$1"
+    if [[ ! -f "$base" ]]; then
+        echo "ROCm base Dockerfile not found: $base" >&2
+        return 2
+    fi
+    ROCM_STACK_BASE_DOCKERFILE="$base"
+    ROCM_STACK_DOCKERFILE="${base%_base}"
+
+    local sdk
+    sdk=$(sed -nE 's/^ARG ROCM_SDK_VERSION=([0-9.]+).*/\1/p' "$base" | head -1)
+    if [[ -n "$sdk" ]]; then
+        ROCM_STACK_THEROCK=1
+        ROCM_STACK_VERSION="$sdk"
+    else
+        ROCM_STACK_THEROCK=0
+        # BASE_IMAGE format: rocm/dev-ubuntu-22.04:7.2.3-complete -> 7.2.3
+        ROCM_STACK_VERSION=$(sed -nE 's/^ARG BASE_IMAGE=.*:([0-9]+(\.[0-9]+)+).*/\1/p' "$base" | head -1)
+    fi
+    if [[ -z "$ROCM_STACK_VERSION" ]]; then
+        echo "Could not determine the ROCm version from $base" >&2
+        return 2
+    fi
+
+    local major minor
+    IFS=. read -r major minor _ <<< "$ROCM_STACK_VERSION"
+    ROCM_STACK_VARIANT="rocm${major}${minor}"
+
+    ROCM_STACK_IMAGE_KEY=$(sha256sum "$base" | cut -c1-16)
+    local inputs
+    inputs=$(grep -E '^COPY ' "$base" | grep -v -- '--from' | awk '{print $2}' || true)
+    if [[ -z "$inputs" ]]; then
+        ROCM_STACK_WHEEL_KEY="$ROCM_STACK_IMAGE_KEY"
+    else
+        # shellcheck disable=SC2086
+        ROCM_STACK_WHEEL_KEY=$(sha256sum "$base" $inputs | sha256sum | cut -c1-16)
+    fi
+
+    local ecr="public.ecr.aws/q9t5s3a7/vllm-release-repo"
+    ROCM_STACK_ECR_BASE="${ecr}:${ROCM_STACK_IMAGE_KEY}-${ROCM_STACK_VARIANT}-base"
+    ROCM_STACK_ECR_IMAGE="${ecr}:${BUILDKITE_COMMIT:-unknown}-${ROCM_STACK_VARIANT}"
+    ROCM_STACK_BASE_WHEELS_DIR="artifacts/${ROCM_STACK_VARIANT}-base-wheels"
+    ROCM_STACK_VLLM_WHEEL_DIR="artifacts/${ROCM_STACK_VARIANT}-vllm-wheel"
+
+    export ROCM_STACK_BASE_DOCKERFILE ROCM_STACK_DOCKERFILE ROCM_STACK_THEROCK \
+        ROCM_STACK_VERSION ROCM_STACK_VARIANT ROCM_STACK_IMAGE_KEY \
+        ROCM_STACK_WHEEL_KEY ROCM_STACK_ECR_BASE ROCM_STACK_ECR_IMAGE \
+        ROCM_STACK_BASE_WHEELS_DIR ROCM_STACK_VLLM_WHEEL_DIR
+}
+
+if [[ $# -gt 0 ]]; then
+    rocm_stack_init "$1"
+fi

.buildkite/scripts/upload-rocm-wheels.sh

@@ -4,21 +4,31 @@
 #
 # Upload ROCm wheels to S3 with proper index generation
 #
+# Usage: upload-rocm-wheels.sh <base-dockerfile>
+#   base-dockerfile  the ROCm stack, e.g. docker/Dockerfile.rocm_base (see rocm/stack.sh)
+#
 # Required environment variables:
 #   AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY (or IAM role)
 #   S3_BUCKET (default: vllm-wheels)
 #
 # S3 path structure:
-#   s3://vllm-wheels/rocm/{commit}/     - All wheels for this commit
+#   s3://vllm-wheels/rocm/{commit}/<variant>-wheels/ - This stack's wheels for this commit
-#   s3://vllm-wheels/rocm/nightly/      - Index pointing to latest nightly
+#   s3://vllm-wheels/rocm/{commit}/<variant>/   - Index per ROCm variant (e.g. rocm100)
-#   s3://vllm-wheels/rocm/{version}/    - Index for release versions
+#   s3://vllm-wheels/rocm/nightly/<variant>/    - Index pointing to latest nightly
+#   s3://vllm-wheels/rocm/{version}/<variant>/  - Index for release versions
+# The top-level index.html at each prefix lists every variant present, since
+# several ROCm stacks publish into the same prefixes.
 
 set -ex
 
 # ======== Configuration ========
 BUCKET="${S3_BUCKET:-vllm-wheels}"
+# shellcheck source=.buildkite/scripts/rocm/stack.sh
+source "$(dirname "${BASH_SOURCE[0]}")/rocm/stack.sh" "${1:?Usage: $0 <base-dockerfile>}"
 ROCM_SUBPATH="rocm/${BUILDKITE_COMMIT}"
-S3_COMMIT_PREFIX="s3://$BUCKET/$ROCM_SUBPATH/"
+WHEEL_DIRS="$ROCM_STACK_BASE_WHEELS_DIR $ROCM_STACK_VLLM_WHEEL_DIR"
+WHEEL_SUBPATH="$ROCM_SUBPATH/${ROCM_STACK_VARIANT}-wheels"
+S3_WHEEL_PREFIX="s3://$BUCKET/$WHEEL_SUBPATH/"
 INDICES_OUTPUT_DIR="rocm-indices"
 
 echo "========================================"
@@ -49,9 +59,11 @@ source .buildkite/scripts/lib/manylinux.sh
 # ======== Part 1: Collect and prepare wheels ========
 
 # Collect all wheels
+rm -rf all-rocm-wheels "$INDICES_OUTPUT_DIR"
 mkdir -p all-rocm-wheels
-cp artifacts/rocm-base-wheels/*.whl all-rocm-wheels/ 2>/dev/null || true
+for dir in $WHEEL_DIRS; do
-cp artifacts/rocm-vllm-wheel/*.whl all-rocm-wheels/ 2>/dev/null || true
+    cp "$dir"/*.whl all-rocm-wheels/ 2>/dev/null || true
+done
 
 WHEEL_COUNT=$(find all-rocm-wheels -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l)
 echo "Total wheels to upload: $WHEEL_COUNT"
@@ -84,9 +96,9 @@ ls -lh all-rocm-wheels/
 # ======== Part 2: Upload wheels to S3 ========
 
 echo ""
-echo "Uploading wheels to $S3_COMMIT_PREFIX"
+echo "Uploading wheels to $S3_WHEEL_PREFIX"
 for wheel in all-rocm-wheels/*.whl; do
-    aws s3 cp "$wheel" "$S3_COMMIT_PREFIX"
+    aws s3 cp "$wheel" "$S3_WHEEL_PREFIX"
 done
 
 # ======== Part 3: Generate and upload indices ========
@@ -95,28 +107,63 @@ done
 echo ""
 echo "Generating indices..."
 obj_json="rocm-objects.json"
-aws s3api list-objects-v2 --bucket "$BUCKET" --prefix "$ROCM_SUBPATH/" --delimiter / --output json > "$obj_json"
+aws s3api list-objects-v2 --bucket "$BUCKET" --prefix "$WHEEL_SUBPATH/" --delimiter / --output json > "$obj_json"
 
 mkdir -p "$INDICES_OUTPUT_DIR"
 
 # Use the existing generate-nightly-index.py
 # HACK: Replace regex module with stdlib re (same as CUDA script)
 sed -i 's/import regex as re/import re/g' .buildkite/scripts/generate-nightly-index.py
+sed -i 's/import regex as re/import re/g' tools/vllm-rocm/therock_wheels.py
+sed -i 's/import pybase64 as base64/import base64/g' tools/vllm-rocm/therock_wheels.py
 
+INDEX_ARGS=(--version "$ROCM_SUBPATH" --wheel-dir "${WHEEL_SUBPATH#rocm/}")
+# TheRock's SDK, device kernels and torchvision/torchaudio are linked, not hosted
+if [[ "$ROCM_STACK_THEROCK" == "1" ]]; then
+    $PYTHON tools/vllm-rocm/therock_wheels.py external-links \
+        --dockerfile "$ROCM_STACK_BASE_DOCKERFILE" \
+        --torch-wheel "$(ls "$ROCM_STACK_BASE_WHEELS_DIR"/torch-*.whl)" > therock-external-links.txt
+    cat therock-external-links.txt
+    INDEX_ARGS+=(--external-links therock-external-links.txt)
+fi
 $PYTHON .buildkite/scripts/generate-nightly-index.py \
-    --version "$ROCM_SUBPATH" \
+    "${INDEX_ARGS[@]}" \
     --current-objects "$obj_json" \
     --output-dir "$INDICES_OUTPUT_DIR" \
     --comment "ROCm commit $BUILDKITE_COMMIT"
 
-# Upload indices to commit directory
+# Upload this build's variant indices under an S3 prefix (e.g. rocm/nightly), then
-echo "Uploading indices to $S3_COMMIT_PREFIX"
+# rebuild that prefix's top-level project list from every variant now present.
-aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "$S3_COMMIT_PREFIX"
+publish_indices() {
+    local prefix="$1"
+    local listing variants merged
+    echo "Uploading indices to s3://$BUCKET/$prefix/"
+    aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/$prefix/" --exclude "index.html"
+    listing=$(aws s3api list-objects-v2 --bucket "$BUCKET" --prefix "$prefix/" \
+        --delimiter / --query 'CommonPrefixes[].Prefix' --output text)
+    variants=$(tr '\t' '\n' <<< "$listing" | sed -n "s|^$prefix/\(rocm[0-9]*\)/$|\1|p" | sort -u)
+    merged=$(mktemp)
+    {
+        echo '<!DOCTYPE html>'
+        echo '<html>'
+        echo '  <meta name="pypi:repository-version" content="1.0">'
+        echo '  <body>'
+        for variant in $variants; do
+            echo "    <a href=\"$variant/\">$variant/</a><br/>"
+        done
+        echo '  </body>'
+        echo '</html>'
+    } > "$merged"
+    echo "Variants under $prefix/: $(echo "$variants" | tr '\n' ' ')"
+    aws s3 cp "$merged" "s3://$BUCKET/$prefix/index.html"
+    rm -f "$merged"
+}
+
+publish_indices "$ROCM_SUBPATH"
 
 # Only scheduled nightly builds should update the moving nightly index.
 if [[ "${NIGHTLY:-0}" == "1" ]]; then
-    echo "Updating rocm/nightly/ index..."
+    publish_indices "rocm/nightly"
-    aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/rocm/nightly/"
 fi
 
 # Extract version from vLLM wheel and update version-specific index
@@ -129,8 +176,7 @@ if [ -n "$VLLM_WHEEL" ]; then
     echo "Pure version: $PURE_VERSION"
 
     if [[ "$VERSION" != *"dev"* ]]; then
-        echo "Updating rocm/$PURE_VERSION/ index..."
+        publish_indices "rocm/$PURE_VERSION"
-        aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/rocm/$PURE_VERSION/"
     fi
 fi
 
@@ -142,14 +188,14 @@ echo "ROCm Wheel Upload Complete!"
 echo "========================================"
 echo ""
 echo "Wheels available at:"
-echo "  s3://$BUCKET/$ROCM_SUBPATH/"
+echo "  s3://$BUCKET/$WHEEL_SUBPATH/"
 echo ""
 echo "Install command (by commit):"
-echo "  pip install vllm --extra-index-url https://${BUCKET}.s3.amazonaws.com/$ROCM_SUBPATH/"
+echo "  pip install vllm --extra-index-url https://${BUCKET}.s3.amazonaws.com/$ROCM_SUBPATH/$ROCM_STACK_VARIANT/"
 echo ""
 if [[ "${NIGHTLY:-0}" == "1" ]]; then
     echo "Install command (nightly):"
-    echo "  pip install vllm --extra-index-url https://${BUCKET}.s3.amazonaws.com/rocm/nightly/"
+    echo "  pip install vllm --extra-index-url https://${BUCKET}.s3.amazonaws.com/rocm/nightly/$ROCM_STACK_VARIANT/"
 fi
 echo ""
 echo "Wheel count: $WHEEL_COUNT"

docker/Dockerfile.rock_base

@@ -1,453 +0,0 @@
-ARG BASE_IMAGE=ubuntu:22.04
-ARG PYTHON_VERSION=3.12
-ARG TORCH_VERSION=2.12.0+rocm10.0.0
-ARG TORCHVISION_VERSION=0.27.0+rocm10.0.0
-ARG TORCHAUDIO_VERSION=2.11.0+rocm10.0.0
-ARG TRITON_BRANCH="669b31ac" # release/internal/3.8.x as of 09/29/2026
-ARG TRITON_VERSION=3.8.0+git669b31ac
-ARG TRITON_REPO="https://github.com/ROCm/triton.git"
-ARG FA_BRANCH="0e60e394"
-ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git"
-ARG AITER_BRANCH="v0.1.25"
-ARG AITER_REPO="https://github.com/ROCm/aiter.git"
-ARG MORI_BRANCH="v1.2.3.post1"
-ARG MORI_REPO="https://github.com/ROCm/mori.git"
-ARG ROCM_SYSTEMS_REPO="https://github.com/ROCm/rocm-systems.git"
-ARG ROCM_RUNTIME_COMMIT="25e14349f2606a6b95c4ccd45eb92a1835646cf2"
-
-
-# TheRock configuration
-ARG SITE_PACKAGES=/usr/local/lib/python${PYTHON_VERSION}/dist-packages
-ARG ROCM_RELEASE_WHEELS_MULTIARCH_URL="https://stable.repo.amd.com/rocm/whl-next/" # For pre-built GPU images, packages, etc.
-ARG ROCM_NIGHTLY_WHEELS_MULTIARCH_URL="https://nightly.repo.amd.com/rocm/whl-next/" # For torch, triton, etc. wheels with latest fixes.
-ARG ROCM_SDK_VERSION=10.0.0
-
-# Build configuration
-ARG PYTORCH_ROCM_ARCH="gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1200;gfx1201;gfx1150;gfx1151"
-ARG AITER_ROCM_ARCH="gfx942;gfx950"
-
-# Sccache configuration (only used in release pipeline)
-ARG USE_SCCACHE
-ARG SCCACHE_DOWNLOAD_URL
-ARG SCCACHE_ENDPOINT
-ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
-ARG SCCACHE_REGION_NAME=us-west-2
-ARG SCCACHE_S3_NO_CREDENTIALS=0
-
-FROM ${BASE_IMAGE} AS base
-
-ARG PYTORCH_ROCM_ARCH
-ENV PYTORCH_ROCM_ARCH="${PYTORCH_ROCM_ARCH}"
-ARG AITER_ROCM_ARCH
-ENV AITER_ROCM_ARCH="${AITER_ROCM_ARCH}"
-
-ARG SITE_PACKAGES
-ENV SITE_PACKAGES=${SITE_PACKAGES}
-ENV SDK_DEV="$SITE_PACKAGES/_rocm_sdk_devel" \
-    SDK_CORE="$SITE_PACKAGES/_rocm_sdk_core"
-ENV ROCM_PATH="$SDK_DEV" \
-    ROCM_HOME="$SDK_DEV" \
-    HIP_PATH="$SDK_DEV" \
-    HIP_CLANG_PATH="$SDK_DEV/lib/llvm/bin" \
-    HIP_DEVICE_LIB_PATH="$SDK_CORE/lib/llvm/amdgcn/bitcode"
-ENV LD_LIBRARY_PATH="$SDK_DEV/lib:$SDK_DEV/lib/rocm_sysdeps/lib:$SDK_CORE/lib:$SDK_CORE/lib/rocm_sysdeps/lib"
-ENV PATH="$SDK_DEV/bin:$SDK_DEV/lib/llvm/bin:$SDK_CORE/bin:$PATH"
-
-# Required for RCCL in ROCm7.1
-ENV HSA_NO_SCRATCH_RECLAIM=1
-
-ARG PYTHON_VERSION
-ENV PYTHON_VERSION=${PYTHON_VERSION}
-
-RUN mkdir -p /app
-WORKDIR /app
-ENV DEBIAN_FRONTEND=noninteractive
-
-# Install Python and other dependencies
-RUN apt-get update -y \
-    && apt-get install -y software-properties-common build-essential git curl sudo vim less libgfortran5 libopenmpi-dev libpci-dev liblzma-dev pkg-config libdrm-dev libnuma-dev kmod \
-    && for i in 1 2 3; do \
-        add-apt-repository -y ppa:deadsnakes/ppa && break || \
-        { echo "Attempt $i failed, retrying in 5s..."; sleep 5; }; \
-    done \
-    && apt-get update -y \
-    && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv \
-       python${PYTHON_VERSION}-lib2to3 python-is-python3 \
-    && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
-    && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \
-    && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \
-    && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \
-    && python3 --version && python3 -m pip --version
-
-RUN pip install -U packaging 'cmake<4' ninja wheel 'setuptools<80' pybind11==3.0.4 Cython
-RUN apt-get update && apt-get install -y libjpeg-dev libsox-dev libsox-fmt-all sox && rm -rf /var/lib/apt/lists/*
-
-# Install ROCm SDK via TheRock
-ARG ROCM_RELEASE_WHEELS_MULTIARCH_URL
-ARG ROCM_NIGHTLY_WHEELS_MULTIARCH_URL
-ARG ROCM_SDK_VERSION
-RUN pip install --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \
-    --extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \
-    "rocm[libraries,devel]${ROCM_SDK_VERSION:+==${ROCM_SDK_VERSION}}" \
-    && rocm-sdk init
-
-# Install PyTorch wheels (nightly for latest torch, release as fallback)
-ARG TORCH_VERSION
-ARG TORCHAUDIO_VERSION
-ARG TORCHVISION_VERSION
-RUN pip install \
-    --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \
-    --extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \
-    "torch==${TORCH_VERSION}" \
-    "torchaudio==${TORCHAUDIO_VERSION}" \
-    "torchvision==${TORCHVISION_VERSION}"
-
-# Install the AMD-compiled device kernels corresponding to PYTORCH_ROCM_ARCH:
-
-# Possible values that can be in PYTORCH_ROCM_ARCH:
-#
-# gfx90a (MI210/250), gfx942 (MI300), gfx950 (MI350), gfx1100 (7900XTX),
-# gfx1101 (7800XT), gfx1200 (9070XT), gfx1201 (9070), gfx1150/1151 (Strix APU)
-
-# For each arch in PYTORCH_ROCM_ARCH, install the precompiled GPU code that
-# PyTorch needs to run on that specific chip.
-SHELL ["/bin/bash", "-c"]
-RUN TORCH_VER=$(python3 -c "import torch; print(torch.__version__)") \
-    && for arch in $(echo "${PYTORCH_ROCM_ARCH}" | tr ';' ' '); do \
-        echo "Installing device packages for ${arch}" \
-        && pip install --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \
-            --extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \
-            amd-torch-device-${arch}==${TORCH_VER} \
-            rocm-sdk-device-${arch}==${ROCM_SDK_VERSION}; \
-    done
-
-# Remove gfx90a xnack kpacks (ASAN artifacts that cause hipErrorInvalidImage on MI250)
-RUN rm -f "${SITE_PACKAGES}/torch/.kpack/torch_gfx90a:xnack+.kpack" \
-         "${SITE_PACKAGES}/torch/.kpack/torch_gfx90a:xnack-.kpack"
-
-RUN cd $SITE_PACKAGES/_rocm_sdk_core/share/amd_smi && pip install .
-ENV PYTHONPATH=$SITE_PACKAGES/_rocm_sdk_core/share/amd_smi
-
-# TheRock runtime fixes
-RUN if [ -f "${SITE_PACKAGES}/rocm_sdk/__init__.py" ]; then \
-        sed -i 's/rtld_global: bool = True/rtld_global: bool = False/g' "${SITE_PACKAGES}/rocm_sdk/__init__.py"; \
-    fi \
-    && if [ -f "${SITE_PACKAGES}/_rocm_sdk_core/lib/librocm_smi64.so.1" ] \
-       && [ ! -e "${ROCM_PATH}/lib/librocm_smi64.so.7" ]; then \
-        ln -sfn "${SITE_PACKAGES}/_rocm_sdk_core/lib/librocm_smi64.so.1" "${ROCM_PATH}/lib/librocm_smi64.so.7"; \
-    fi \
-    && printf '%s\n%s\n' "${ROCM_PATH}/lib" "${SITE_PACKAGES}/_rocm_sdk_core/lib" > /etc/ld.so.conf.d/rocm-pip.conf \
-    && for f in ${SITE_PACKAGES}/_rocm_sdk_core/lib/libamdhip64.so*; do \
-        bn=$(basename "$f"); \
-        if [ -f "${ROCM_PATH}/lib/${bn}" ] && [ -f "$f" ] && [ ! -L "$f" ]; then \
-            rm -f "$f"; \
-            ln -sfn "${ROCM_PATH}/lib/${bn}" "$f"; \
-        fi; \
-    done \
-    && for f in librocprofiler-sdk.so.1 librocprofiler-register.so.0 \
-              librocprofiler-sdk-roctx.so.1; do \
-        if [ -e "${SITE_PACKAGES}/_rocm_sdk_core/lib/${f}" ] \
-           && [ -e "${SITE_PACKAGES}/_rocm_sdk_devel/lib/${f}" ]; then \
-            rm -f "${SITE_PACKAGES}/_rocm_sdk_core/lib/${f}"; \
-            ln -sfn "${SITE_PACKAGES}/_rocm_sdk_devel/lib/${f}" \
-                "${SITE_PACKAGES}/_rocm_sdk_core/lib/${f}"; \
-        fi; \
-    done \
-    && ldconfig
-
-# Install sccache if USE_SCCACHE is enabled (for release builds)
-ARG USE_SCCACHE
-ARG SCCACHE_DOWNLOAD_URL
-ARG SCCACHE_ENDPOINT
-ARG SCCACHE_BUCKET_NAME
-ARG SCCACHE_REGION_NAME
-ARG SCCACHE_S3_NO_CREDENTIALS
-RUN if [ "$USE_SCCACHE" = "1" ]; then \
-        echo "Installing sccache..." \
-        && SCCACHE_ARCH="x86_64" \
-        && SCCACHE_VERSION="v0.8.1" \
-        && SCCACHE_DL_URL="${SCCACHE_DOWNLOAD_URL:-https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}-unknown-linux-musl.tar.gz}" \
-        && curl -L -o /tmp/sccache.tar.gz ${SCCACHE_DL_URL} \
-        && tar -xzf /tmp/sccache.tar.gz -C /tmp \
-        && mv /tmp/sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}-unknown-linux-musl/sccache /usr/bin/sccache \
-        && chmod +x /usr/bin/sccache \
-        && rm -rf /tmp/sccache.tar.gz /tmp/sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}-unknown-linux-musl \
-        && sccache --version; \
-    fi
-
-# Setup sccache for HIP compilation via HIP_CLANG_PATH
-# This creates wrapper scripts in a separate directory and points HIP to use them
-# This avoids modifying the original ROCm binaries which can break detection
-# NOTE: HIP_CLANG_PATH is NOT set as ENV to avoid affecting downstream images (Dockerfile.rocm)
-# Instead, each build stage should export HIP_CLANG_PATH=/opt/sccache-wrappers if USE_SCCACHE=1
-RUN if [ "$USE_SCCACHE" = "1" ]; then \
-        echo "Setting up sccache wrappers for HIP compilation..." \
-        && mkdir -p /opt/sccache-wrappers \
-        && printf '#!/bin/bash\nexec sccache %s/lib/llvm/bin/clang++ "$@"\n' "${ROCM_PATH}" > /opt/sccache-wrappers/clang++ \
-        && chmod +x /opt/sccache-wrappers/clang++ \
-        && printf '#!/bin/bash\nexec sccache %s/lib/llvm/bin/clang "$@"\n' "${ROCM_PATH}" > /opt/sccache-wrappers/clang \
-        && chmod +x /opt/sccache-wrappers/clang \
-        && echo "sccache wrappers created in /opt/sccache-wrappers"; \
-    fi
-
-ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET_NAME}}
-ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}}
-ENV SCCACHE_S3_NO_CREDENTIALS=${USE_SCCACHE:+${SCCACHE_S3_NO_CREDENTIALS}}
-ENV SCCACHE_IDLE_TIMEOUT=${USE_SCCACHE:+0}
-
-
-###
-### MORI Build
-###
-FROM base AS build_mori
-ARG MORI_BRANCH
-ARG MORI_REPO
-RUN git clone ${MORI_REPO}
-RUN cd mori \
-    && git checkout ${MORI_BRANCH} \
-    && git submodule update --init --recursive \
-    && ln -sfn ${ROCM_PATH} /opt/rocm \
-    && pip install "setuptools_scm>=6.2" \
-    && CMAKE_PREFIX_PATH="${ROCM_PATH}:${ROCM_PATH}/lib/rocm_sysdeps" \
-       python3 setup.py bdist_wheel --dist-dir=dist && ls /app/mori/dist/*.whl
-RUN mkdir -p /app/install && cp /app/mori/dist/*.whl /app/install
-
-###
-### FlashAttention Build
-###
-FROM base AS build_fa
-ARG FA_BRANCH
-ARG FA_REPO
-ARG USE_SCCACHE
-RUN git clone ${FA_REPO}
-RUN cd flash-attention \
-    && git checkout ${FA_BRANCH} \
-    && git submodule update --init \
-    && if [ "$USE_SCCACHE" = "1" ]; then \
-           export HIP_CLANG_PATH=/opt/sccache-wrappers \
-           && sccache --show-stats; \
-       fi \
-    && GPU_ARCHS=$(echo ${PYTORCH_ROCM_ARCH} | sed -e 's/;gfx1[0-9]\{3\}//g') python3 setup.py bdist_wheel --dist-dir=dist \
-    && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
-RUN mkdir -p /app/install && cp /app/flash-attention/dist/*.whl /app/install
-
-
-###
-### AITER Build
-###
-FROM base AS build_aiter
-ARG AITER_BRANCH
-ARG AITER_REPO
-ARG USE_SCCACHE
-RUN git clone --recursive --branch ${AITER_BRANCH} ${AITER_REPO}
-RUN cd aiter \
-    && git submodule update --init --recursive \
-    && pip install -r requirements.txt
-# AITER_USE_SYSTEM_TRITON keeps setup.py from resolving its own triton, which would
-# displace the pinned one already installed in base.
-RUN pip install pyyaml && cd aiter \
-    && if [ "$USE_SCCACHE" = "1" ]; then \
-           export HIP_CLANG_PATH=/opt/sccache-wrappers \
-           && sccache --show-stats; \
-       fi \
-    && PREBUILD_KERNELS=1 AITER_USE_SYSTEM_TRITON=1 GPU_ARCHS=${AITER_ROCM_ARCH} python3 setup.py bdist_wheel --dist-dir=dist \
-    && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi \
-    && ls /app/aiter/dist/*.whl
-RUN mkdir -p /app/install && cp /app/aiter/dist/*.whl /app/install
-
-
-###
-### Triton Build
-###
-FROM base AS build_triton
-ARG TRITON_BRANCH
-ARG TRITON_REPO
-RUN git clone ${TRITON_REPO}
-RUN cd triton \
-    && git checkout ${TRITON_BRANCH} \
-    && if [ ! -f setup.py ]; then cd python; fi \
-    && python3 setup.py bdist_wheel --dist-dir=dist \
-    && mkdir -p /app/install && cp dist/*.whl /app/install
-
-
-###
-### ROCr + CLR build
-###
-# Brings in multi-stream perf improvements; drop after these arrive in an official release
-FROM base AS build_rocm_runtime
-ARG ROCM_SYSTEMS_REPO
-ARG ROCM_RUNTIME_COMMIT
-# libgl-dev: rocclr does find_package(OpenGL REQUIRED) unconditionally and the wheel image has
-# no GL headers. The link must not pick up a libGL DT_NEEDED, which is asserted after the build.
-RUN apt-get update \
-    && apt-get install -y --no-install-recommends g++ libdw-dev xxd libgl-dev libglx-dev \
-    && rm -rf /var/lib/apt/lists/*
-RUN pip install --no-cache-dir CppHeaderParser
-RUN git init -q /src && cd /src \
-    && git remote add origin ${ROCM_SYSTEMS_REPO} \
-    && git config core.sparseCheckout true \
-    && git config remote.origin.promisor true \
-    && git config remote.origin.partialclonefilter blob:none \
-    && git sparse-checkout init --cone \
-    && git sparse-checkout set projects/rocr-runtime projects/clr projects/hip shared cmake \
-    && git fetch --filter=blob:none --depth 1 origin ${ROCM_RUNTIME_COMMIT} \
-    && git checkout -q FETCH_HEAD
-# libelf is taken from the SDK's own vendored sysdeps rather than apt -- the wheel image has no
-# /usr/include/libelf.h -- so ROCr links against the copy the wheel itself was built against.
-ENV SYSDEPS="$SDK_DEV/lib/rocm_sysdeps"
-# ROCr first, and installed into $SDK_DEV, because the clr build below needs its headers.
-RUN cd /src/projects/rocr-runtime \
-    && cmake -B build -G Ninja \
-        -DCMAKE_BUILD_TYPE=Release \
-        -DCMAKE_PREFIX_PATH="$SDK_DEV;$SYSDEPS" \
-        -DCMAKE_INSTALL_PREFIX="$SDK_DEV" \
-        -DLIBELF_INCLUDE_DIRS="$SYSDEPS/include" \
-        -DLIBELF_LIBRARIES="$SYSDEPS/lib/libelf.so" \
-    && cmake --build build --parallel 16 \
-    && cmake --install build \
-    && cmake --install build --prefix /rocr-install --strip
-RUN cd /src/projects/clr \
-    && cmake -B build -G Ninja \
-        -DCMAKE_BUILD_TYPE=Release \
-        -DCLR_BUILD_HIP=ON \
-        -DCLR_BUILD_OCL=OFF \
-        -DROCM_KPACK_ENABLED=ON \
-        -DHIP_COMMON_DIR=/src/projects/hip \
-        -DROCM_PATH="$SDK_DEV" \
-        -DCMAKE_PREFIX_PATH="$SDK_DEV;$SYSDEPS" \
-        -DCMAKE_INSTALL_PREFIX="$SDK_DEV" \
-        -DHIP_LLVM_ROOT="$SDK_DEV/lib/llvm" \
-    && cmake --build build --parallel 16 \
-    && cmake --install build --prefix /clr-install --strip
-RUN mkdir -p /staging/lib \
-    && cp -P /rocr-install/lib/libhsa-runtime64.so* /staging/lib/ \
-    && cp -P /clr-install/lib/libamdhip64.so* /staging/lib/ \
-    && ls -l /staging/lib
-
-
-###
-### Final Build
-###
-
-# Wheel release stage -
-# only includes dependencies used by wheel release pipeline
-FROM base AS debs_wheel_release
-RUN mkdir /app/debs
-RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-
-FROM base AS debs
-RUN mkdir /app/debs
-RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-RUN --mount=type=bind,from=build_mori,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-
-FROM base AS final
-ARG TRITON_VERSION
-
-# torch 2.12 implicitly attaches rocprofiler-sdk; its queue interposition hangs at HSA
-# teardown, so disable it. TODO: drop once ROCm's stock rocprofiler carries the fix.
-ENV ROCPROFILER_QUEUE_INTERPOSITION=0
-
-COPY --from=build_rocm_runtime /staging/lib/ ${SDK_DEV}/lib/
-# Hard link the patched library onto every stock file name, in both SDK trees, so that the
-# SONAME lookup and the hard coded paths all resolve to it. Hard links rather than symlinks
-# because ldconfig rewrites SONAME symlinks but cannot re-point an inode. $SDK_CORE keeps its
-# own libhsa-runtime64 copy, which is on LD_LIBRARY_PATH and in ld.so.conf; relinking it is
-# belt and braces rather than strictly required, but it leaves no stock ROCr in the image.
-RUN set -eux; \
-    cd "$SDK_DEV/lib"; \
-    ours_hsa="$(readlink -f libhsa-runtime64.so.1)"; \
-    ours_hip="$(readlink -f libamdhip64.so.7)"; \
-    for f in "$SDK_DEV"/lib/libhsa-runtime64.so.1* "$SDK_CORE"/lib/libhsa-runtime64.so.1* \
-             "$SDK_DEV"/lib/libamdhip64.so.7* "$SDK_CORE"/lib/libamdhip64.so.7*; do \
-        [ -f "$f" ] && [ ! -L "$f" ] || continue; \
-        case "${f##*/}" in libhsa*) ours="$ours_hsa";; *) ours="$ours_hip";; esac; \
-        [ "$f" -ef "$ours" ] || ln -f "$ours" "$f"; \
-    done
-
-RUN ldconfig
-
-# This is to prevent resolvers from checking METADATA and RECORD and clobbering
-# the Triton source build we installed. Constraints files are unable to help
-# with this. However, these entries are optional and can be blanked which
-# solves the problem.
-RUN TORCH_DIST_INFO=$(ls -d ${SITE_PACKAGES}/torch-*.dist-info) \
-    && grep -q '^Requires-Dist: triton' "$TORCH_DIST_INFO/METADATA" \
-    && sed -i '/^Requires-Dist: triton/d' "$TORCH_DIST_INFO/METADATA" \
-    && sed -i "s|^\(${TORCH_DIST_INFO##*/}/METADATA\),.*|\1,,|" "$TORCH_DIST_INFO/RECORD"
-
-RUN --mount=type=bind,from=debs,src=/app/debs,target=/install \
-    pip install /install/*.whl \
-    && ! pip check 2>&1 | grep -iE '^(torch|triton) ' \
-    && test "$(pip show triton | awk '/^Version:/ {print $2}')" = "${TRITON_VERSION}" \
-    # Create a constraints file to protect the versions we need, especially
-    # triton, requirements will try to reinstall repeatedly.
-    && pip list --format=freeze \
-        | grep -E '^(torch|triton|torchaudio|torchvision|pybind11)==' > /etc/rocm-constraints.txt \
-    && echo "ROCm constraints:" && cat /etc/rocm-constraints.txt \
-    && test "$(wc -l < /etc/rocm-constraints.txt)" -eq 5 \
-    && grep -q '^pybind11==3\.0\.' /etc/rocm-constraints.txt \
-    && python3 -c "import torch; assert torch.version.hip, f'non-ROCm torch: {torch.__version__}'; print(f'torch={torch.__version__}')" \
-    && python3 -c "import triton; print(f'triton={triton.__version__}')" \
-    && pip list 2>/dev/null | grep -iE '^(torch|triton) '
-
-ARG BASE_IMAGE
-ARG PYTHON_VERSION
-ARG ROCM_SDK_VERSION
-ARG TORCH_VERSION
-ARG TORCHAUDIO_VERSION
-ARG TORCHVISION_VERSION
-ARG PYTORCH_ROCM_ARCH
-ARG AITER_ROCM_ARCH
-ARG TRITON_BRANCH
-ARG TRITON_REPO
-ARG FA_BRANCH
-ARG FA_REPO
-ARG AITER_BRANCH
-ARG AITER_REPO
-ARG MORI_BRANCH
-ARG MORI_REPO
-ARG ROCM_SYSTEMS_REPO
-ARG ROCM_RUNTIME_COMMIT
-ARG TRITON_VERSION
-# Record hashes for Triton and pybind11 so we can verify later.
-RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \
-    && echo "PYTHON_VERSION: ${PYTHON_VERSION}" >> /app/versions.txt \
-    && echo "ROCM_SDK_VERSION: ${ROCM_SDK_VERSION}" >> /app/versions.txt \
-    && echo "TORCH_VERSION: ${TORCH_VERSION}" >> /app/versions.txt \
-    && echo "TORCHAUDIO_VERSION: ${TORCHAUDIO_VERSION}" >> /app/versions.txt \
-    && echo "TORCHVISION_VERSION: ${TORCHVISION_VERSION}" >> /app/versions.txt \
-    && echo "PYTORCH_ROCM_ARCH: ${PYTORCH_ROCM_ARCH}" >> /app/versions.txt \
-    && echo "AITER_ROCM_ARCH: ${AITER_ROCM_ARCH}" >> /app/versions.txt \
-    && echo "TRITON_BRANCH: ${TRITON_BRANCH}" >> /app/versions.txt \
-    && echo "TRITON_REPO: ${TRITON_REPO}" >> /app/versions.txt \
-    && echo "FA_BRANCH: ${FA_BRANCH}" >> /app/versions.txt \
-    && echo "FA_REPO: ${FA_REPO}" >> /app/versions.txt \
-    && echo "AITER_BRANCH: ${AITER_BRANCH}" >> /app/versions.txt \
-    && echo "AITER_REPO: ${AITER_REPO}" >> /app/versions.txt \
-    && echo "MORI_BRANCH: ${MORI_BRANCH}" >> /app/versions.txt \
-    && echo "MORI_REPO: ${MORI_REPO}" >> /app/versions.txt \
-    && echo "ROCM_SYSTEMS_REPO: ${ROCM_SYSTEMS_REPO}" >> /app/versions.txt \
-    && echo "ROCM_RUNTIME_COMMIT: ${ROCM_RUNTIME_COMMIT}" >> /app/versions.txt \
-    # Fingerprint our triton so Dockerfile.rocm can prove it survived the build.
-    && echo "TRITON_VERSION: $(pip show triton | awk '/^Version:/ {print $2}')" >> /app/versions.txt \
-    && echo "TRITON_SHA256: $(find $(python3 -c 'import triton, os; print(os.path.dirname(triton.__file__))') \
-        -type f \( -name '*.py' -o -name '*.so' \) | sort | xargs sha256sum | sha256sum | cut -d' ' -f1)" \
-        >> /app/versions.txt \
-    && echo "PYBIND11_VERSION: $(pip show pybind11 | awk '/^Version:/ {print $2}')" >> /app/versions.txt \
-    && echo "PYBIND11_INTERNALS_VERSION: $(grep -rhE '^#[[:space:]]*define PYBIND11_INTERNALS_VERSION' \
-        $(python3 -c 'import pybind11; print(pybind11.get_include())') | awk '{print $NF}')" \
-        >> /app/versions.txt \
-    && echo "PYBIND11_SHA256: $(find $(python3 -c 'import pybind11; print(pybind11.get_include())') \
-        -type f -name '*.h' | sort | xargs sha256sum | sha256sum | cut -d' ' -f1)" \
-        >> /app/versions.txt

docker/Dockerfile.rocm

@@ -280,8 +280,9 @@ COPY --from=fetch_vllm \
     ${COMMON_WORKDIR}/vllm/requirements/rocm.txt \
     requirements/
 
+# Use constraints file to protect triton, pybind11, and torch dependencies.
 RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
-    uv pip install --system -r requirements/rocm.txt
+    uv pip install --system -c /etc/rocm-constraints.txt -r requirements/rocm.txt
 
 # Fetch only the pinned ROCm Triton kernels package. Keeping this before the
 # vLLM source COPYs makes the download reusable when native sources change.
@@ -322,8 +323,9 @@ ENV SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+rocm.csrc.build"
 
 RUN --mount=type=bind,source=pyproject.toml,target=${COMMON_WORKDIR}/vllm/pyproject.toml \
     --mount=type=cache,id=vllm-rocm-ccache,target=/root/.cache/ccache \
-    grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \
+    { grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \
         cmake/external_projects/triton_kernels.cmake \
+        || echo "WARNING: Dockerfile.rocm ROCM_TRITON_KERNELS_COMMIT (${ROCM_TRITON_KERNELS_COMMIT}) does not match cmake TRITON_KERNELS_TAG; the Docker build uses the Dockerfile pin, cmake governs only non-Docker builds"; } \
     && export CCACHE_BASEDIR="$PWD" \
     && echo "=== ccache stats before ROCm native build ===" \
     && (ccache --show-stats || true) \
@@ -366,7 +368,8 @@ COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/requirements /requirements
 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks
 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tests /tests
 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/examples /examples
-COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools /tools
+COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools/benchmark_helion_kernels.py /tools/benchmark_helion_kernels.py
+COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh
 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile /docker/Dockerfile
 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.cpu /docker/Dockerfile.cpu
 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/
@@ -386,7 +389,6 @@ ARG NIXL_BRANCH="1683cf3b7f3d11674c03c5e861cea22339876c96"
 ARG NIXL_REPO="https://github.com/ai-dynamo/nixl.git"
 ARG UCX_BRANCH="96e58a16039f6d7d213bc967b8069238742c5194"
 ARG UCX_REPO="https://github.com/openucx/ucx.git"
-ENV ROCM_PATH=/opt/rocm
 ENV UCX_HOME=/usr/local/ucx
 ENV NIXL_HOME=/usr/local/nixl
 
@@ -541,7 +543,7 @@ RUN --mount=type=tmpfs,target=/tmp/lmcache-wheel \
     c_ops="$(find /tmp/lmcache-wheel -type f -name 'c_ops*.so')"; \
     test "$(find /tmp/lmcache-wheel -type f -name 'c_ops*.so' | wc -l)" -eq 1; \
     expected="$(printf '%s' "${LMCACHE_ROCM_ARCH}" | tr ';' '\n' | sort -u)"; \
-    actual="$(/opt/rocm/llvm/bin/llvm-objdump --offloading "${c_ops}" \
+    actual="$(llvm-objdump --offloading "${c_ops}" \
         | grep -oE 'gfx[0-9a-z]+' | sort -u)"; \
     if [ "${actual}" != "${expected}" ]; then \
         printf 'LMCache code objects mismatch\nexpected:\n%s\nactual:\n%s\n' \
@@ -549,32 +551,10 @@ RUN --mount=type=tmpfs,target=/tmp/lmcache-wheel \
         exit 1; \
     fi
 
-# ROCShmem build stage - split from DeepEP so changing DEEPEP_BRANCH does not
+# DeepEP build stage. ROCShmem is provided by TheRock SDK
-# invalidate the slow ROCShmem build.
+# ROCSHMEM_DIR set to the SDK install path so DeepEP finds the headers, etc.
-FROM base AS build_rocshmem
+FROM base AS build_deepep
-ARG ROCSHMEM_BRANCH="f0acb0c6"
+ENV ROCSHMEM_DIR=${ROCM_PATH}
-ARG ROCSHMEM_REPO="https://github.com/ROCm/rocm-systems.git"
-# DeepEP only supports gfx942 and gfx950; build ROCShmem for the same set so
-# it can be linked against DeepEP without arch mismatches.
-ARG DEEPEP_ROCM_ARCH="gfx942;gfx950"
-ENV ROCM_PATH=/opt/rocm
-ENV ROCSHMEM_DIR=/opt/rocshmem
-
-RUN --mount=type=cache,target=/root/.cache/ccache \
-    git clone --no-checkout --filter=blob:none ${ROCSHMEM_REPO} \
- && cd rocm-systems \
- && git sparse-checkout set --cone projects/rocshmem \
- && git checkout ${ROCSHMEM_BRANCH} \
- && mkdir -p projects/rocshmem/build \
- && cd projects/rocshmem/build \
- && CC="ccache gcc" CXX="ccache g++" INSTALL_PREFIX=${ROCSHMEM_DIR} \
-    bash ../scripts/build_configs/all_backends \
-      -DROCM_PATH=${ROCM_PATH} \
-      -DGPU_TARGETS="${DEEPEP_ROCM_ARCH}" \
-      -DUSE_EXTERNAL_MPI=OFF
-
-# DeepEP build stage - depends on ROCShmem, builds the HIP kernel wheel.
-FROM build_rocshmem AS build_deepep
 ARG DEEPEP_BRANCH="0f63d3e9"
 ARG DEEPEP_REPO="https://github.com/ROCm/DeepEP.git"
 ARG DEEPEP_NIC="cx7"
@@ -693,8 +673,9 @@ RUN if [ "$GIT_REPO_CHECK" != "0" ]; then \
 
 # Install the packaging helpers before modifying requirements/rocm.txt.
 RUN --mount=type=cache,target=/root/.cache/uv \
-    grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \
+    { grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \
         vllm/cmake/external_projects/triton_kernels.cmake \
+        || echo "WARNING: Dockerfile.rocm ROCM_TRITON_KERNELS_COMMIT (${ROCM_TRITON_KERNELS_COMMIT}) does not match cmake TRITON_KERNELS_TAG; the Docker build uses the Dockerfile pin, cmake governs only non-Docker builds"; } \
     && cd vllm \
     && uv pip install --system setuptools_scm regex
 
@@ -750,7 +731,8 @@ COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/requirements /requir
 COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks
 COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tests /tests
 COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/examples /examples
-COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools /tools
+COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools/benchmark_helion_kernels.py /tools/benchmark_helion_kernels.py
+COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh
 COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile /docker/Dockerfile
 COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.cpu /docker/Dockerfile.cpu
 COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/
@@ -795,9 +777,6 @@ RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \
     --mount=type=bind,from=build_deepep,src=/app/deep_install,target=/deep_install \
     uv pip install --system /nixl_install/*.whl /deep_install/*.whl
 
-# Copy ROCShmem runtime libraries.
-COPY --from=build_rocshmem /opt/rocshmem /opt/rocshmem
-
 # RDMA userspace libraries plus FFmpeg dev libs needed by torchcodec.
 RUN timeout --kill-after=30s 5m apt-get update -q -y --error-on=any \
     -o Acquire::Retries=3 \
@@ -830,7 +809,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \
 # Pre-install shared ROCm runtime dependencies.
 COPY requirements/common.txt requirements/rocm.txt /tmp/ci-base-requirements/
 RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip install --system -r /tmp/ci-base-requirements/rocm.txt \
+    uv pip install --system -c /etc/rocm-constraints.txt \
+        -r /tmp/ci-base-requirements/rocm.txt \
     && rm -rf /tmp/ci-base-requirements
 
 # Enable fast and less brittle model downloads in tests.
@@ -908,8 +888,6 @@ RUN mkdir src && mv vllm src/vllm
 # Catch GPU<->CPU syncs in execute_model/sample_tokens during tests.
 ENV VLLM_GPU_SYNC_CHECK=error
 
-# Run the structural smoke test inside BuildKit, where the test image layers
-# are already available. Export only a build-scoped success marker.
 FROM test AS test_smoke
 ARG ROCM_SMOKE_ID
 RUN --network=none \
@@ -956,15 +934,37 @@ RUN --mount=type=bind,from=export_vllm,src=/,target=/install \
     --mount=type=cache,target=/root/.cache/uv \
     cd /install \
     && echo "opentelemetry-api<=1.40.0" > /tmp/otel-constraints.txt \
-    && uv pip install --system -c /tmp/otel-constraints.txt -r requirements/rocm.txt \
+    && uv pip install --system -c /etc/rocm-constraints.txt -c /tmp/otel-constraints.txt -r requirements/rocm.txt \
     && pip uninstall -y vllm \
-    && uv pip install --system -c /tmp/otel-constraints.txt *.whl \
+    && uv pip install --system -c /etc/rocm-constraints.txt -c /tmp/otel-constraints.txt *.whl \
     && rm /tmp/otel-constraints.txt
 
 # Install NIXL ROCm wheel
 RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \
     uv pip install --system /nixl_install/*.whl
 
+# Recompute the base's triton fingerprint to detect resolver interference.
+RUN EXPECTED=$(grep '^TRITON_SHA256: ' /app/versions.txt | cut -d' ' -f2) \
+    && ACTUAL=$(find $(python3 -c 'import triton, os; print(os.path.dirname(triton.__file__))') \
+        -type f \( -name '*.py' -o -name '*.so' \) | sort | xargs sha256sum | sha256sum | cut -d' ' -f1) \
+    && echo "triton sha256 expected=${EXPECTED} actual=${ACTUAL}" \
+    && test -n "${EXPECTED}" \
+    && test "${EXPECTED}" = "${ACTUAL}" \
+    && python3 -c "import torch; assert torch.version.hip, f'non-ROCm torch: {torch.__version__}'; print(f'torch={torch.__version__}')"
+
+# Recompute the base's pybind11 fingerprint to detect resolver interference.
+RUN EXPECTED=$(grep '^PYBIND11_SHA256: ' /app/versions.txt | cut -d' ' -f2) \
+    && EXPECTED_ABI=$(grep '^PYBIND11_INTERNALS_VERSION: ' /app/versions.txt | cut -d' ' -f2) \
+    && INC=$(python3 -c 'import pybind11; print(pybind11.get_include())') \
+    && ACTUAL=$(find "${INC}" -type f -name '*.h' | sort | xargs sha256sum | sha256sum | cut -d' ' -f1) \
+    && ACTUAL_ABI=$(grep -rhE '^#[[:space:]]*define PYBIND11_INTERNALS_VERSION' "${INC}" | awk '{print $NF}') \
+    && echo "pybind11 $(pip show pybind11 | awk '/^Version:/ {print $2}')" \
+        "sha256 expected=${EXPECTED} actual=${ACTUAL}" \
+        "internals expected=${EXPECTED_ABI} actual=${ACTUAL_ABI}" \
+    && test -n "${EXPECTED}" \
+    && test "${EXPECTED}" = "${ACTUAL}" \
+    && test "${EXPECTED_ABI}" = "${ACTUAL_ABI}"
+
 ARG COMMON_WORKDIR
 ARG BASE_IMAGE
 ARG NIC_BACKEND

docker/Dockerfile.rocm_72

@@ -3,7 +3,7 @@ ARG REMOTE_VLLM="0"
 ARG VLLM_REPO="https://github.com/vllm-project/vllm.git"
 ARG VLLM_BRANCH="main"
 ARG COMMON_WORKDIR=/app
-ARG BASE_IMAGE=rocm/vllm-dev:base
+ARG BASE_IMAGE=vllm/vllm-openai-rocm:base-nightly-rocm72
 ARG CI_BASE_IMAGE=rocm/vllm-dev:ci_base
 ARG ROCM_TRITON_KERNELS_COMMIT=669b31acc1dd1b3fd93286afd5db67f65d9f7557
 ARG INSTALL_LMCACHE=false
@@ -46,7 +46,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 # Note: mold is installed but not set as the system default linker because
 # some packages use JIT compilation at runtime with flags mold does not support.
 # Build stages opt in via LDFLAGS="-fuse-ld=mold".
-# Remove sccache only if not using sccache (it exists in base image from Dockerfile.rocm_base)
+# Remove sccache only if not using sccache (it exists in base image from Dockerfile.rocm_72_base)
 ARG USE_SCCACHE
 RUN if [ "$USE_SCCACHE" != "1" ]; then \
         apt-get purge -y sccache || true; \
@@ -280,9 +280,8 @@ COPY --from=fetch_vllm \
     ${COMMON_WORKDIR}/vllm/requirements/rocm.txt \
     requirements/
 
-# Use constraints file to protect triton, pybind11, and torch dependencies.
 RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \
-    uv pip install --system -c /etc/rocm-constraints.txt -r requirements/rocm.txt
+    uv pip install --system -r requirements/rocm.txt
 
 # Fetch only the pinned ROCm Triton kernels package. Keeping this before the
 # vLLM source COPYs makes the download reusable when native sources change.
@@ -323,9 +322,8 @@ ENV SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+rocm.csrc.build"
 
 RUN --mount=type=bind,source=pyproject.toml,target=${COMMON_WORKDIR}/vllm/pyproject.toml \
     --mount=type=cache,id=vllm-rocm-ccache,target=/root/.cache/ccache \
-    { grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \
+    grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \
         cmake/external_projects/triton_kernels.cmake \
-        || echo "WARNING: Dockerfile.rock ROCM_TRITON_KERNELS_COMMIT (${ROCM_TRITON_KERNELS_COMMIT}) does not match cmake TRITON_KERNELS_TAG; the Docker build uses the Dockerfile pin, cmake governs only non-Docker builds"; } \
     && export CCACHE_BASEDIR="$PWD" \
     && echo "=== ccache stats before ROCm native build ===" \
     && (ccache --show-stats || true) \
@@ -368,12 +366,11 @@ COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/requirements /requirements
 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks
 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tests /tests
 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/examples /examples
-COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools/benchmark_helion_kernels.py /tools/benchmark_helion_kernels.py
+COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools /tools
-COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh
 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile /docker/Dockerfile
 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.cpu /docker/Dockerfile.cpu
-COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/
+COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm_72 /docker/
-COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm_base /docker/Dockerfile.rocm_base
+COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm_72_base /docker/
 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/ci-rocm.hcl /docker/ci-rocm.hcl
 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/docker-bake.hcl /docker/docker-bake.hcl
 COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/docker-bake-rocm.hcl /docker/docker-bake-rocm.hcl
@@ -389,6 +386,7 @@ ARG NIXL_BRANCH="1683cf3b7f3d11674c03c5e861cea22339876c96"
 ARG NIXL_REPO="https://github.com/ai-dynamo/nixl.git"
 ARG UCX_BRANCH="96e58a16039f6d7d213bc967b8069238742c5194"
 ARG UCX_REPO="https://github.com/openucx/ucx.git"
+ENV ROCM_PATH=/opt/rocm
 ENV UCX_HOME=/usr/local/ucx
 ENV NIXL_HOME=/usr/local/nixl
 
@@ -543,7 +541,7 @@ RUN --mount=type=tmpfs,target=/tmp/lmcache-wheel \
     c_ops="$(find /tmp/lmcache-wheel -type f -name 'c_ops*.so')"; \
     test "$(find /tmp/lmcache-wheel -type f -name 'c_ops*.so' | wc -l)" -eq 1; \
     expected="$(printf '%s' "${LMCACHE_ROCM_ARCH}" | tr ';' '\n' | sort -u)"; \
-    actual="$(llvm-objdump --offloading "${c_ops}" \
+    actual="$(/opt/rocm/llvm/bin/llvm-objdump --offloading "${c_ops}" \
         | grep -oE 'gfx[0-9a-z]+' | sort -u)"; \
     if [ "${actual}" != "${expected}" ]; then \
         printf 'LMCache code objects mismatch\nexpected:\n%s\nactual:\n%s\n' \
@@ -551,10 +549,32 @@ RUN --mount=type=tmpfs,target=/tmp/lmcache-wheel \
         exit 1; \
     fi
 
-# DeepEP build stage. ROCShmem is provided by TheRock SDK
+# ROCShmem build stage - split from DeepEP so changing DEEPEP_BRANCH does not
-# ROCSHMEM_DIR set to the SDK install path so DeepEP finds the headers, etc.
+# invalidate the slow ROCShmem build.
-FROM base AS build_deepep
+FROM base AS build_rocshmem
-ENV ROCSHMEM_DIR=${ROCM_PATH}
+ARG ROCSHMEM_BRANCH="f0acb0c6"
+ARG ROCSHMEM_REPO="https://github.com/ROCm/rocm-systems.git"
+# DeepEP only supports gfx942 and gfx950; build ROCShmem for the same set so
+# it can be linked against DeepEP without arch mismatches.
+ARG DEEPEP_ROCM_ARCH="gfx942;gfx950"
+ENV ROCM_PATH=/opt/rocm
+ENV ROCSHMEM_DIR=/opt/rocshmem
+
+RUN --mount=type=cache,target=/root/.cache/ccache \
+    git clone --no-checkout --filter=blob:none ${ROCSHMEM_REPO} \
+ && cd rocm-systems \
+ && git sparse-checkout set --cone projects/rocshmem \
+ && git checkout ${ROCSHMEM_BRANCH} \
+ && mkdir -p projects/rocshmem/build \
+ && cd projects/rocshmem/build \
+ && CC="ccache gcc" CXX="ccache g++" INSTALL_PREFIX=${ROCSHMEM_DIR} \
+    bash ../scripts/build_configs/all_backends \
+      -DROCM_PATH=${ROCM_PATH} \
+      -DGPU_TARGETS="${DEEPEP_ROCM_ARCH}" \
+      -DUSE_EXTERNAL_MPI=OFF
+
+# DeepEP build stage - depends on ROCShmem, builds the HIP kernel wheel.
+FROM build_rocshmem AS build_deepep
 ARG DEEPEP_BRANCH="0f63d3e9"
 ARG DEEPEP_REPO="https://github.com/ROCm/DeepEP.git"
 ARG DEEPEP_NIC="cx7"
@@ -571,8 +591,8 @@ RUN --mount=type=cache,target=/root/.cache/ccache \
  && git checkout ${DEEPEP_BRANCH} \
  && LDFLAGS="-fuse-ld=mold" MAX_JOBS="${MAX_JOBS:-$(nproc)}" python3 setup.py --variant rocm --rocm-explicit-ctx --nic ${DEEPEP_NIC} --rocm-gfx942-fp8-fnuz-max 224 bdist_wheel --dist-dir=/app/deep_install
 
-# MoRI runtime dependencies live in Dockerfile.rocm so NIC backend changes do
+# MoRI runtime dependencies live in Dockerfile.rocm_72 so NIC backend changes do
-# not force users to rebuild the long-lived Dockerfile.rocm_base image.
+# not force users to rebuild the long-lived Dockerfile.rocm_72_base image.
 FROM base AS mori_base
 ARG NIC_BACKEND
 ARG AINIC_VERSION
@@ -673,9 +693,8 @@ RUN if [ "$GIT_REPO_CHECK" != "0" ]; then \
 
 # Install the packaging helpers before modifying requirements/rocm.txt.
 RUN --mount=type=cache,target=/root/.cache/uv \
-    { grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \
+    grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \
         vllm/cmake/external_projects/triton_kernels.cmake \
-        || echo "WARNING: Dockerfile.rock ROCM_TRITON_KERNELS_COMMIT (${ROCM_TRITON_KERNELS_COMMIT}) does not match cmake TRITON_KERNELS_TAG; the Docker build uses the Dockerfile pin, cmake governs only non-Docker builds"; } \
     && cd vllm \
     && uv pip install --system setuptools_scm regex
 
@@ -731,12 +750,11 @@ COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/requirements /requir
 COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks
 COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tests /tests
 COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/examples /examples
-COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools/benchmark_helion_kernels.py /tools/benchmark_helion_kernels.py
+COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools /tools
-COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh
 COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile /docker/Dockerfile
 COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.cpu /docker/Dockerfile.cpu
-COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/
+COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm_72 /docker/
-COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm_base /docker/Dockerfile.rocm_base
+COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm_72_base /docker/
 COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/ci-rocm.hcl /docker/ci-rocm.hcl
 COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/docker-bake.hcl /docker/docker-bake.hcl
 COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/docker-bake-rocm.hcl /docker/docker-bake-rocm.hcl
@@ -777,6 +795,9 @@ RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \
     --mount=type=bind,from=build_deepep,src=/app/deep_install,target=/deep_install \
     uv pip install --system /nixl_install/*.whl /deep_install/*.whl
 
+# Copy ROCShmem runtime libraries.
+COPY --from=build_rocshmem /opt/rocshmem /opt/rocshmem
+
 # RDMA userspace libraries plus FFmpeg dev libs needed by torchcodec.
 RUN timeout --kill-after=30s 5m apt-get update -q -y --error-on=any \
     -o Acquire::Retries=3 \
@@ -809,8 +830,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
 # Pre-install shared ROCm runtime dependencies.
 COPY requirements/common.txt requirements/rocm.txt /tmp/ci-base-requirements/
 RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip install --system -c /etc/rocm-constraints.txt \
+    uv pip install --system -r /tmp/ci-base-requirements/rocm.txt \
-        -r /tmp/ci-base-requirements/rocm.txt \
     && rm -rf /tmp/ci-base-requirements
 
 # Enable fast and less brittle model downloads in tests.
@@ -888,6 +908,8 @@ RUN mkdir src && mv vllm src/vllm
 # Catch GPU<->CPU syncs in execute_model/sample_tokens during tests.
 ENV VLLM_GPU_SYNC_CHECK=error
 
+# Run the structural smoke test inside BuildKit, where the test image layers
+# are already available. Export only a build-scoped success marker.
 FROM test AS test_smoke
 ARG ROCM_SMOKE_ID
 RUN --network=none \
@@ -934,37 +956,15 @@ RUN --mount=type=bind,from=export_vllm,src=/,target=/install \
     --mount=type=cache,target=/root/.cache/uv \
     cd /install \
     && echo "opentelemetry-api<=1.40.0" > /tmp/otel-constraints.txt \
-    && uv pip install --system -c /etc/rocm-constraints.txt -c /tmp/otel-constraints.txt -r requirements/rocm.txt \
+    && uv pip install --system -c /tmp/otel-constraints.txt -r requirements/rocm.txt \
     && pip uninstall -y vllm \
-    && uv pip install --system -c /etc/rocm-constraints.txt -c /tmp/otel-constraints.txt *.whl \
+    && uv pip install --system -c /tmp/otel-constraints.txt *.whl \
     && rm /tmp/otel-constraints.txt
 
 # Install NIXL ROCm wheel
 RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \
     uv pip install --system /nixl_install/*.whl
 
-# Recompute the base's triton fingerprint to detect resolver interference.
-RUN EXPECTED=$(grep '^TRITON_SHA256: ' /app/versions.txt | cut -d' ' -f2) \
-    && ACTUAL=$(find $(python3 -c 'import triton, os; print(os.path.dirname(triton.__file__))') \
-        -type f \( -name '*.py' -o -name '*.so' \) | sort | xargs sha256sum | sha256sum | cut -d' ' -f1) \
-    && echo "triton sha256 expected=${EXPECTED} actual=${ACTUAL}" \
-    && test -n "${EXPECTED}" \
-    && test "${EXPECTED}" = "${ACTUAL}" \
-    && python3 -c "import torch; assert torch.version.hip, f'non-ROCm torch: {torch.__version__}'; print(f'torch={torch.__version__}')"
-
-# Recompute the base's pybind11 fingerprint to detect resolver interference.
-RUN EXPECTED=$(grep '^PYBIND11_SHA256: ' /app/versions.txt | cut -d' ' -f2) \
-    && EXPECTED_ABI=$(grep '^PYBIND11_INTERNALS_VERSION: ' /app/versions.txt | cut -d' ' -f2) \
-    && INC=$(python3 -c 'import pybind11; print(pybind11.get_include())') \
-    && ACTUAL=$(find "${INC}" -type f -name '*.h' | sort | xargs sha256sum | sha256sum | cut -d' ' -f1) \
-    && ACTUAL_ABI=$(grep -rhE '^#[[:space:]]*define PYBIND11_INTERNALS_VERSION' "${INC}" | awk '{print $NF}') \
-    && echo "pybind11 $(pip show pybind11 | awk '/^Version:/ {print $2}')" \
-        "sha256 expected=${EXPECTED} actual=${ACTUAL}" \
-        "internals expected=${EXPECTED_ABI} actual=${ACTUAL_ABI}" \
-    && test -n "${EXPECTED}" \
-    && test "${EXPECTED}" = "${ACTUAL}" \
-    && test "${EXPECTED_ABI}" = "${ACTUAL_ABI}"
-
 ARG COMMON_WORKDIR
 ARG BASE_IMAGE
 ARG NIC_BACKEND

docker/Dockerfile.rocm_72_base

@@ -0,0 +1,479 @@
+ARG BASE_IMAGE=rocm/dev-ubuntu-22.04:7.2.3-complete
+ARG TRITON_BRANCH="669b31a" # release/internal/3.8.x as of 09/17
+ARG TRITON_REPO="https://github.com/ROCm/triton.git"
+ARG PYTORCH_BRANCH="733fca1" # release/2.13 as of 09/17
+ARG PYTORCH_REPO="https://github.com/ROCm/pytorch.git"
+ARG PYTORCH_VISION_BRANCH="v0.28.0"
+ARG PYTORCH_VISION_REPO="https://github.com/pytorch/vision.git"
+ARG PYTORCH_AUDIO_BRANCH="v2.11.0"
+ARG PYTORCH_AUDIO_REPO="https://github.com/pytorch/audio.git"
+ARG FA_BRANCH="0e60e394"
+ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git"
+ARG AITER_BRANCH="v0.1.25"
+ARG AITER_REPO="https://github.com/ROCm/aiter.git"
+ARG MORI_BRANCH="v1.2.3.post1"
+ARG MORI_REPO="https://github.com/ROCm/mori.git"
+ARG ROCM_SYSTEMS_REPO="https://github.com/ROCm/rocm-systems.git"
+ARG ROCPROFILER_SDK_COMMIT="2b22ab0195cc1461cd9abf3b969e9dd7c10af350"
+ARG ROCM_RUNTIME_COMMIT="af70edc91f1dc5d228fd4c076df99e65e49c4521"
+
+# Sccache configuration (only used in release pipeline)
+ARG USE_SCCACHE
+ARG SCCACHE_DOWNLOAD_URL
+ARG SCCACHE_ENDPOINT
+ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
+ARG SCCACHE_REGION_NAME=us-west-2
+ARG SCCACHE_S3_NO_CREDENTIALS=0
+
+
+FROM ${BASE_IMAGE} AS base
+
+ENV PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ENV ROCM_PATH=/opt/rocm
+ENV LD_LIBRARY_PATH=/opt/rocm/lib:/usr/local/lib:
+
+ARG PYTORCH_ROCM_ARCH=gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1200;gfx1201;gfx1150;gfx1151
+ENV PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH}
+ENV AITER_ROCM_ARCH=gfx942;gfx950
+# Note: Do not set MORI_GPU_ARCHS here, it is automatically inferred at runtime
+
+# Required for RCCL in ROCm7.1
+ENV HSA_NO_SCRATCH_RECLAIM=1
+
+ARG PYTHON_VERSION=3.12
+ENV PYTHON_VERSION=${PYTHON_VERSION}
+
+RUN mkdir -p /app
+WORKDIR /app
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Install Python and other dependencies (plus gcc-13, selected later in build_pytorch).
+RUN apt-get update -y \
+    && apt-get install -y software-properties-common git curl sudo vim less libgfortran5 libopenmpi-dev libpci-dev liblzma-dev pkg-config \
+    && for i in 1 2 3; do \
+        add-apt-repository -y ppa:deadsnakes/ppa && add-apt-repository -y ppa:ubuntu-toolchain-r/test && break || \
+        { echo "Attempt $i failed, retrying in 5s..."; sleep 5; }; \
+    done \
+    && apt-get update -y \
+    && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv \
+       python${PYTHON_VERSION}-lib2to3 python-is-python3 gcc-13 g++-13 \
+    && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
+    && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \
+    && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \
+    && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \
+    && python3 --version && python3 -m pip --version
+
+RUN pip install -U packaging 'cmake<4' ninja wheel 'setuptools<80' 'pybind11<3.1.0' Cython
+RUN apt-get update && apt-get install -y libjpeg-dev libsox-dev libsox-fmt-all sox && rm -rf /var/lib/apt/lists/*
+
+# Install sccache if USE_SCCACHE is enabled (for release builds)
+ARG USE_SCCACHE
+ARG SCCACHE_DOWNLOAD_URL
+ARG SCCACHE_ENDPOINT
+ARG SCCACHE_BUCKET_NAME
+ARG SCCACHE_REGION_NAME
+ARG SCCACHE_S3_NO_CREDENTIALS
+RUN if [ "$USE_SCCACHE" = "1" ]; then \
+        echo "Installing sccache..." \
+        && SCCACHE_ARCH="x86_64" \
+        && SCCACHE_VERSION="v0.8.1" \
+        && SCCACHE_DL_URL="${SCCACHE_DOWNLOAD_URL:-https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}-unknown-linux-musl.tar.gz}" \
+        && curl -L -o /tmp/sccache.tar.gz ${SCCACHE_DL_URL} \
+        && tar -xzf /tmp/sccache.tar.gz -C /tmp \
+        && mv /tmp/sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}-unknown-linux-musl/sccache /usr/bin/sccache \
+        && chmod +x /usr/bin/sccache \
+        && rm -rf /tmp/sccache.tar.gz /tmp/sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}-unknown-linux-musl \
+        && sccache --version; \
+    fi
+
+# Setup sccache for HIP compilation via HIP_CLANG_PATH
+# This creates wrapper scripts in a separate directory and points HIP to use them
+# This avoids modifying the original ROCm binaries which can break detection
+# NOTE: HIP_CLANG_PATH is NOT set as ENV to avoid affecting downstream images (Dockerfile.rocm_72)
+# Instead, each build stage should export HIP_CLANG_PATH=/opt/sccache-wrappers if USE_SCCACHE=1
+RUN if [ "$USE_SCCACHE" = "1" ]; then \
+        echo "Setting up sccache wrappers for HIP compilation..." \
+        && mkdir -p /opt/sccache-wrappers \
+        && printf '#!/bin/bash\nexec sccache /opt/rocm/lib/llvm/bin/clang++ "$@"\n' > /opt/sccache-wrappers/clang++ \
+        && chmod +x /opt/sccache-wrappers/clang++ \
+        && printf '#!/bin/bash\nexec sccache /opt/rocm/lib/llvm/bin/clang "$@"\n' > /opt/sccache-wrappers/clang \
+        && chmod +x /opt/sccache-wrappers/clang \
+        && echo "sccache wrappers created in /opt/sccache-wrappers"; \
+    fi
+
+# Set sccache environment variables only when USE_SCCACHE=1
+# This prevents S3 config from leaking into images when sccache is not used
+ARG USE_SCCACHE
+ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET_NAME}}
+ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}}
+ENV SCCACHE_S3_NO_CREDENTIALS=${USE_SCCACHE:+${SCCACHE_S3_NO_CREDENTIALS}}
+ENV SCCACHE_IDLE_TIMEOUT=${USE_SCCACHE:+0}
+
+
+###
+### Triton Build
+###
+FROM base AS build_triton
+ARG TRITON_BRANCH
+ARG TRITON_REPO
+RUN git clone ${TRITON_REPO}
+RUN cd triton \
+    && git checkout ${TRITON_BRANCH} \
+    && git config --global user.email "you@example.com" && git config --global user.name "Your Name" \
+    && if [ ! -f setup.py ]; then cd python; fi \
+    && MAX_JOBS=16 python3 setup.py bdist_wheel --dist-dir=dist \
+    && mkdir -p /app/install && cp dist/*.whl /app/install
+
+
+###
+### AMD SMI Build
+###
+FROM base AS build_amdsmi
+RUN cd /opt/rocm/share/amd_smi \
+    && pip wheel . --wheel-dir=dist
+RUN mkdir -p /app/install && cp /opt/rocm/share/amd_smi/dist/*.whl /app/install
+
+
+###
+### Pytorch build
+###
+FROM base AS build_pytorch
+ARG PYTORCH_BRANCH
+ARG PYTORCH_REPO
+ARG USE_SCCACHE
+
+RUN apt-get update && apt-get install -y pkg-config liblzma-dev
+# torch 2.12 emits float->__bf16 libcalls (__truncsfbf2) absent from gcc-11 libgcc; build with gcc-13.
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 --slave /usr/bin/g++ g++ /usr/bin/g++-13
+RUN git clone ${PYTORCH_REPO} pytorch
+RUN cd pytorch && git checkout ${PYTORCH_BRANCH}
+RUN cd pytorch \
+    && pip install -r requirements.txt && git submodule update --init --recursive
+RUN cd pytorch && python3 tools/amd_build/build_amd.py \
+    && if [ "$USE_SCCACHE" = "1" ]; then \
+           export HIP_CLANG_PATH=/opt/sccache-wrappers \
+           && export CMAKE_C_COMPILER_LAUNCHER=sccache \
+           && export CMAKE_CXX_COMPILER_LAUNCHER=sccache \
+           && sccache --show-stats; \
+       fi \
+    && CMAKE_PREFIX_PATH=$(python3 -c 'import sys; print(sys.prefix)') python3 setup.py bdist_wheel --dist-dir=dist \
+    && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
+RUN mkdir -p /app/install && cp /app/pytorch/dist/*.whl /app/install
+
+
+###
+### Shared PyTorch runtime for dependent builds
+###
+FROM base AS build_pytorch_runtime
+RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 --slave /usr/bin/g++ g++ /usr/bin/g++-13
+RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install,ro \
+    pip install /install/*.whl
+
+
+###
+### TorchVision build
+###
+FROM build_pytorch_runtime AS build_torchvision
+ARG PYTORCH_VISION_BRANCH
+ARG PYTORCH_VISION_REPO
+ARG USE_SCCACHE
+
+RUN git clone ${PYTORCH_VISION_REPO} vision
+RUN cd vision && git checkout ${PYTORCH_VISION_BRANCH} \
+    && if [ "$USE_SCCACHE" = "1" ]; then \
+           export HIP_CLANG_PATH=/opt/sccache-wrappers \
+           && export CMAKE_C_COMPILER_LAUNCHER=sccache \
+           && export CMAKE_CXX_COMPILER_LAUNCHER=sccache; \
+       fi \
+    && python3 setup.py bdist_wheel --dist-dir=dist \
+    && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
+RUN mkdir -p /app/install && cp /app/vision/dist/*.whl /app/install
+
+
+###
+### TorchAudio build
+###
+FROM build_pytorch_runtime AS build_torchaudio
+ARG PYTORCH_AUDIO_BRANCH
+ARG PYTORCH_AUDIO_REPO
+ARG USE_SCCACHE
+
+RUN git clone ${PYTORCH_AUDIO_REPO} audio
+RUN cd audio && git checkout ${PYTORCH_AUDIO_BRANCH} \
+    && git submodule update --init --recursive
+RUN cd audio && pip install -r requirements.txt
+RUN cd audio \
+    && if [ "$USE_SCCACHE" = "1" ]; then \
+           export HIP_CLANG_PATH=/opt/sccache-wrappers \
+           && export CMAKE_C_COMPILER_LAUNCHER=sccache \
+           && export CMAKE_CXX_COMPILER_LAUNCHER=sccache; \
+       fi \
+    && python3 setup.py bdist_wheel --dist-dir=dist \
+    && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
+RUN mkdir -p /app/install && cp /app/audio/dist/*.whl /app/install
+
+
+###
+### MORI Build
+###
+FROM base AS build_mori
+ARG MORI_BRANCH
+ARG MORI_REPO
+RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \
+    pip install /install/*.whl
+RUN git clone ${MORI_REPO}
+RUN cd mori \
+    && git checkout ${MORI_BRANCH} \
+    && git submodule update --init --recursive \
+    && python3 setup.py bdist_wheel --dist-dir=dist && ls /app/mori/dist/*.whl
+RUN mkdir -p /app/install && cp /app/mori/dist/*.whl /app/install
+
+
+###
+### FlashAttention Build
+###
+FROM base AS build_fa
+ARG FA_BRANCH
+ARG FA_REPO
+ARG USE_SCCACHE
+RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \
+    pip install /install/*.whl
+RUN git clone ${FA_REPO}
+RUN cd flash-attention \
+    && git checkout ${FA_BRANCH} \
+    && git submodule update --init \
+    && if [ "$USE_SCCACHE" = "1" ]; then \
+           export HIP_CLANG_PATH=/opt/sccache-wrappers \
+           && sccache --show-stats; \
+       fi \
+    && GPU_ARCHS=$(echo ${PYTORCH_ROCM_ARCH} | sed -e 's/;gfx1[0-9]\{3\}//g') python3 setup.py bdist_wheel --dist-dir=dist \
+    && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
+RUN mkdir -p /app/install && cp /app/flash-attention/dist/*.whl /app/install
+
+
+###
+### AITER Build
+###
+FROM base AS build_aiter
+ARG AITER_BRANCH
+ARG AITER_REPO
+ARG USE_SCCACHE
+RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \
+    pip install /install/*.whl
+RUN git clone --recursive --branch ${AITER_BRANCH} ${AITER_REPO}
+RUN cd aiter \
+    && git submodule update --init --recursive \
+    && pip install -r requirements.txt
+RUN pip install pyyaml && cd aiter \
+    && if [ "$USE_SCCACHE" = "1" ]; then \
+           export HIP_CLANG_PATH=/opt/sccache-wrappers \
+           && sccache --show-stats; \
+       fi \
+    && PREBUILD_KERNELS=1 AITER_USE_SYSTEM_TRITON=1 GPU_ARCHS=${AITER_ROCM_ARCH} python3 setup.py bdist_wheel --dist-dir=dist \
+    && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi \
+    && ls /app/aiter/dist/*.whl
+RUN mkdir -p /app/install && cp /app/aiter/dist/*.whl /app/install
+
+
+###
+### Final Build
+###
+
+# Wheel release stage - 
+# only includes dependencies used by wheel release pipeline
+FROM base AS debs_wheel_release
+RUN mkdir /app/debs
+RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \
+    cp /install/*.whl /app/debs
+RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \
+    cp /install/*.whl /app/debs
+RUN --mount=type=bind,from=build_amdsmi,src=/app/install/,target=/install \
+    cp /install/*.whl /app/debs
+RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \
+    cp /install/*.whl /app/debs
+RUN --mount=type=bind,from=build_torchvision,src=/app/install/,target=/install \
+    cp /install/*.whl /app/debs
+RUN --mount=type=bind,from=build_torchaudio,src=/app/install/,target=/install \
+    cp /install/*.whl /app/debs
+RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
+    cp /install/*.whl /app/debs
+
+# Full debs stage - includes Mori (used by Docker releases)
+FROM base AS debs
+RUN mkdir /app/debs
+RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \
+    cp /install/*.whl /app/debs
+RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \
+    cp /install/*.whl /app/debs
+RUN --mount=type=bind,from=build_amdsmi,src=/app/install/,target=/install \
+    cp /install/*.whl /app/debs
+RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \
+    cp /install/*.whl /app/debs
+RUN --mount=type=bind,from=build_torchvision,src=/app/install/,target=/install \
+    cp /install/*.whl /app/debs
+RUN --mount=type=bind,from=build_torchaudio,src=/app/install/,target=/install \
+    cp /install/*.whl /app/debs
+RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
+    cp /install/*.whl /app/debs
+RUN --mount=type=bind,from=build_mori,src=/app/install/,target=/install \
+    cp /install/*.whl /app/debs
+
+###
+### ROCr + CLR build
+###
+# Pull in updates to ROCr + clr from ROCm 7.2.4 and beyond.
+FROM base AS build_rocm_runtime
+ARG ROCM_SYSTEMS_REPO
+ARG ROCM_RUNTIME_COMMIT
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends g++ \
+       libelf-dev libdrm-dev libnuma-dev libdw-dev xxd \
+    && rm -rf /var/lib/apt/lists/*
+RUN pip install --no-cache-dir CppHeaderParser
+RUN git init -q /src && cd /src \
+    && git remote add origin ${ROCM_SYSTEMS_REPO} \
+    && git config core.sparseCheckout true \
+    && git config remote.origin.promisor true \
+    && git config remote.origin.partialclonefilter blob:none \
+    && git sparse-checkout init --cone \
+    && git sparse-checkout set projects/rocr-runtime projects/clr projects/hip shared cmake \
+    && git fetch --filter=blob:none --depth 1 origin ${ROCM_RUNTIME_COMMIT} \
+    && git checkout -q FETCH_HEAD
+# ROCr's trap_handler and blit_shaders call find_package(Clang/LLVM REQUIRED) only to get
+# IMPORTED executables. The ROCm dev image ships the binaries but no CMake package config, so
+# supply one pointing at them.
+RUN mkdir -p /opt/rocm-cmake-shim \
+    && printf 'if(NOT TARGET clang)\n  add_executable(clang IMPORTED GLOBAL)\n  set_target_properties(clang PROPERTIES IMPORTED_LOCATION "/opt/rocm/llvm/bin/clang")\nendif()\nset(Clang_PACKAGE_VERSION "rocm-image-shim")\n' > /opt/rocm-cmake-shim/ClangConfig.cmake \
+    && printf 'if(NOT TARGET llvm-objcopy)\n  add_executable(llvm-objcopy IMPORTED GLOBAL)\n  set_target_properties(llvm-objcopy PROPERTIES IMPORTED_LOCATION "/opt/rocm/llvm/bin/llvm-objcopy")\nendif()\nset(LLVM_FOUND TRUE)\nset(LLVM_PACKAGE_VERSION "rocm-image-shim")\n' > /opt/rocm-cmake-shim/LLVMConfig.cmake
+# ROCr first, and installed into /opt/rocm, because the clr build below needs its headers
+RUN cd /src/projects/rocr-runtime \
+    && cmake -B build -G Ninja \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_PREFIX_PATH=/opt/rocm \
+        -DCMAKE_INSTALL_PREFIX=/opt/rocm \
+        -DClang_DIR=/opt/rocm-cmake-shim \
+        -DLLVM_DIR=/opt/rocm-cmake-shim \
+    && cmake --build build --parallel 16 \
+    && cmake --install build \
+    && cmake --install build --prefix /rocr-install --strip
+RUN cd /src/projects/clr \
+    && cmake -B build -G Ninja \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DCLR_BUILD_HIP=ON \
+        -DCLR_BUILD_OCL=OFF \
+        -DHIP_COMMON_DIR=/src/projects/hip \
+        -DROCM_PATH=/opt/rocm \
+        -DCMAKE_PREFIX_PATH=/opt/rocm \
+        -DCMAKE_INSTALL_PREFIX=/opt/rocm \
+        -DLLVM_DIR=/opt/rocm-cmake-shim \
+        -DHIP_LLVM_ROOT=/opt/rocm/llvm \
+    && cmake --build build --parallel 16 \
+    && cmake --install build --prefix /clr-install --strip
+RUN mkdir -p /staging/lib \
+    && cp -P /rocr-install/lib/libhsa-runtime64.so* /staging/lib/ \
+    && cp -P /clr-install/lib/libamdhip64.so* /staging/lib/ \
+    && ls -l /staging/lib
+
+# rocprofiler-sdk 1.3.2 fixes torch 2.12 profiler on ROCm 7.2.3
+FROM base AS build_rocprofiler_sdk
+ARG ROCM_SYSTEMS_REPO
+ARG ROCPROFILER_SDK_COMMIT
+ARG USE_SCCACHE
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends libdw-dev libelf-dev libsqlite3-dev libzstd-dev \
+    && rm -rf /var/lib/apt/lists/*
+RUN git init -q /src && cd /src \
+    && git remote add origin ${ROCM_SYSTEMS_REPO} \
+    && git config core.sparseCheckout true \
+    && git config remote.origin.promisor true \
+    && git config remote.origin.partialclonefilter blob:none \
+    && git sparse-checkout init --cone \
+    && git sparse-checkout set projects/rocprofiler-sdk shared cmake \
+    && git fetch --filter=blob:none --depth 1 origin ${ROCPROFILER_SDK_COMMIT} \
+    && git checkout -q FETCH_HEAD
+# Patch the ROCm 7.2.3 build of rocprofiler-sdk to include queue-interposition fixes that are
+# absent from the pinned 7.14 build. This resolves hangs on torch.profiler on ROCm.
+# TODO: Drop these once ${ROCPROFILER_SDK_COMMIT} is bumped to a build that already 
+# contains them, such as ROCm 10.0. See https://github.com/vllm-project/vllm/issues/54087.
+ARG ROCPROFILER_SDK_PR_7796=fae7fd57ffb8a76e811d6410b8a0cee1e94d710a
+ARG ROCPROFILER_SDK_PR_7924=f8db22adbe17686a9b05ae03367dd1ec73011653
+RUN cd /src \
+    && HSA_PATHS='projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/*' \
+    && curl -fsSL -o /tmp/qi-7796.patch "${ROCM_SYSTEMS_REPO%.git}/commit/${ROCPROFILER_SDK_PR_7796}.patch" \
+    && curl -fsSL -o /tmp/qi-7924.patch "${ROCM_SYSTEMS_REPO%.git}/commit/${ROCPROFILER_SDK_PR_7924}.patch" \
+    && git apply -p1 --include="$HSA_PATHS" --verbose /tmp/qi-7796.patch \
+    && git apply -p1 --include="$HSA_PATHS" --verbose /tmp/qi-7924.patch
+RUN cd /src/projects/rocprofiler-sdk \
+    && if [ "$USE_SCCACHE" = "1" ]; then \
+           export CMAKE_C_COMPILER_LAUNCHER=sccache \
+           && export CMAKE_CXX_COMPILER_LAUNCHER=sccache; \
+       fi \
+    && cmake -B build -G Ninja \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_PREFIX_PATH=/opt/rocm \
+        -DCMAKE_INSTALL_PREFIX=/opt/rocm \
+        -DROCPROFILER_BUILD_TESTS=OFF \
+        -DROCPROFILER_BUILD_SAMPLES=OFF \
+        -DROCPROFILER_BUILD_DOCS=OFF \
+    && cmake --build build --parallel "$(nproc)" \
+    && cmake --install build --prefix /staging --strip
+
+FROM base AS final
+RUN --mount=type=bind,from=debs,src=/app/debs,target=/install \
+    pip install /install/*.whl
+
+COPY --from=build_rocprofiler_sdk /staging/ /opt/rocm/
+ENV ROCPROFILER_QUEUE_INTERPOSITION=0
+
+COPY --from=build_rocm_runtime /staging/ /opt/rocm/
+# Hard link the patched library onto every stock file name so that both the
+# SONAME lookup and the hard coded paths resolve to it.
+RUN set -eux; \
+    cd /opt/rocm/lib; \
+    ours_hsa="$(readlink -f libhsa-runtime64.so.1)"; \
+    ours_hip="$(readlink -f libamdhip64.so.7)"; \
+    for f in libhsa-runtime64.so.1.*; do \
+        [ "$f" = "${ours_hsa##*/}" ] || ln -f "$ours_hsa" "$f"; \
+    done; \
+    for f in libamdhip64.so.7.*; do \
+        [ "$f" = "${ours_hip##*/}" ] || ln -f "$ours_hip" "$f"; \
+    done
+
+RUN ldconfig
+
+ARG BASE_IMAGE
+ARG TRITON_BRANCH
+ARG TRITON_REPO
+ARG PYTORCH_BRANCH
+ARG PYTORCH_VISION_BRANCH
+ARG PYTORCH_REPO
+ARG PYTORCH_VISION_REPO
+ARG PYTORCH_AUDIO_BRANCH
+ARG PYTORCH_AUDIO_REPO
+ARG FA_BRANCH
+ARG FA_REPO
+ARG AITER_BRANCH
+ARG AITER_REPO
+ARG MORI_BRANCH
+ARG MORI_REPO
+ARG ROCM_SYSTEMS_REPO
+ARG ROCPROFILER_SDK_COMMIT
+ARG ROCM_RUNTIME_COMMIT
+RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \
+    && echo "ROCM_SYSTEMS_REPO: ${ROCM_SYSTEMS_REPO}" >> /app/versions.txt \
+    && echo "ROCPROFILER_SDK_COMMIT: ${ROCPROFILER_SDK_COMMIT}" >> /app/versions.txt \
+    && echo "ROCM_RUNTIME_COMMIT: ${ROCM_RUNTIME_COMMIT}" >> /app/versions.txt \
+    && echo "TRITON_BRANCH: ${TRITON_BRANCH}" >> /app/versions.txt \
+    && echo "TRITON_REPO: ${TRITON_REPO}" >> /app/versions.txt \
+    && echo "PYTORCH_BRANCH: ${PYTORCH_BRANCH}" >> /app/versions.txt \
+    && echo "PYTORCH_VISION_BRANCH: ${PYTORCH_VISION_BRANCH}" >> /app/versions.txt \
+    && echo "PYTORCH_REPO: ${PYTORCH_REPO}" >> /app/versions.txt \
+    && echo "PYTORCH_VISION_REPO: ${PYTORCH_VISION_REPO}" >> /app/versions.txt \
+    && echo "PYTORCH_AUDIO_BRANCH: ${PYTORCH_AUDIO_BRANCH}" >> /app/versions.txt \
+    && echo "PYTORCH_AUDIO_REPO: ${PYTORCH_AUDIO_REPO}" >> /app/versions.txt \
+    && echo "FA_BRANCH: ${FA_BRANCH}" >> /app/versions.txt \
+    && echo "FA_REPO: ${FA_REPO}" >> /app/versions.txt \
+    && echo "AITER_BRANCH: ${AITER_BRANCH}" >> /app/versions.txt \
+    && echo "AITER_REPO: ${AITER_REPO}" >> /app/versions.txt \
+    && echo "MORI_BRANCH: ${MORI_BRANCH}" >> /app/versions.txt \
+    && echo "MORI_REPO: ${MORI_REPO}" >> /app/versions.txt

docker/Dockerfile.rocm_base

@@ -1,12 +1,11 @@
-ARG BASE_IMAGE=rocm/dev-ubuntu-22.04:7.2.3-complete
+ARG BASE_IMAGE=ubuntu:22.04
-ARG TRITON_BRANCH="669b31a" # release/internal/3.8.x as of 09/17
+ARG PYTHON_VERSION=3.12
+ARG TORCH_VERSION=2.12.0+rocm10.0.0
+ARG TORCHVISION_VERSION=0.27.0+rocm10.0.0
+ARG TORCHAUDIO_VERSION=2.11.0+rocm10.0.0
+ARG TRITON_BRANCH="669b31ac" # release/internal/3.8.x as of 09/29/2026
+ARG TRITON_VERSION=3.8.0+git669b31ac
 ARG TRITON_REPO="https://github.com/ROCm/triton.git"
-ARG PYTORCH_BRANCH="733fca1" # release/2.13 as of 09/17
-ARG PYTORCH_REPO="https://github.com/ROCm/pytorch.git"
-ARG PYTORCH_VISION_BRANCH="v0.28.0"
-ARG PYTORCH_VISION_REPO="https://github.com/pytorch/vision.git"
-ARG PYTORCH_AUDIO_BRANCH="v2.11.0"
-ARG PYTORCH_AUDIO_REPO="https://github.com/pytorch/audio.git"
 ARG FA_BRANCH="0e60e394"
 ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git"
 ARG AITER_BRANCH="v0.1.25"
@@ -14,8 +13,18 @@ ARG AITER_REPO="https://github.com/ROCm/aiter.git"
 ARG MORI_BRANCH="v1.2.3.post1"
 ARG MORI_REPO="https://github.com/ROCm/mori.git"
 ARG ROCM_SYSTEMS_REPO="https://github.com/ROCm/rocm-systems.git"
-ARG ROCPROFILER_SDK_COMMIT="2b22ab0195cc1461cd9abf3b969e9dd7c10af350"
+ARG ROCM_RUNTIME_COMMIT="25e14349f2606a6b95c4ccd45eb92a1835646cf2"
-ARG ROCM_RUNTIME_COMMIT="af70edc91f1dc5d228fd4c076df99e65e49c4521"
+
+
+# TheRock configuration
+ARG SITE_PACKAGES=/usr/local/lib/python${PYTHON_VERSION}/dist-packages
+ARG ROCM_RELEASE_WHEELS_MULTIARCH_URL="https://stable.repo.amd.com/rocm/whl-next/" # For pre-built GPU images, packages, etc.
+ARG ROCM_NIGHTLY_WHEELS_MULTIARCH_URL="https://nightly.repo.amd.com/rocm/whl-next/" # For torch, triton, etc. wheels with latest fixes.
+ARG ROCM_SDK_VERSION=10.0.0
+
+# Build configuration
+ARG PYTORCH_ROCM_ARCH="gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1200;gfx1201;gfx1150;gfx1151"
+ARG AITER_ROCM_ARCH="gfx942;gfx950"
 
 # Sccache configuration (only used in release pipeline)
 ARG USE_SCCACHE
@@ -25,47 +34,128 @@ ARG SCCACHE_BUCKET_NAME=vllm-build-sccache
 ARG SCCACHE_REGION_NAME=us-west-2
 ARG SCCACHE_S3_NO_CREDENTIALS=0
 
-
 FROM ${BASE_IMAGE} AS base
 
-ENV PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ARG PYTORCH_ROCM_ARCH
-ENV ROCM_PATH=/opt/rocm
+ENV PYTORCH_ROCM_ARCH="${PYTORCH_ROCM_ARCH}"
-ENV LD_LIBRARY_PATH=/opt/rocm/lib:/usr/local/lib:
+ARG AITER_ROCM_ARCH
+ENV AITER_ROCM_ARCH="${AITER_ROCM_ARCH}"
 
-ARG PYTORCH_ROCM_ARCH=gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1200;gfx1201;gfx1150;gfx1151
+ARG SITE_PACKAGES
-ENV PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH}
+ENV SITE_PACKAGES=${SITE_PACKAGES}
-ENV AITER_ROCM_ARCH=gfx942;gfx950
+ENV SDK_DEV="$SITE_PACKAGES/_rocm_sdk_devel" \
-# Note: Do not set MORI_GPU_ARCHS here, it is automatically inferred at runtime
+    SDK_CORE="$SITE_PACKAGES/_rocm_sdk_core"
+ENV ROCM_PATH="$SDK_DEV" \
+    ROCM_HOME="$SDK_DEV" \
+    HIP_PATH="$SDK_DEV" \
+    HIP_CLANG_PATH="$SDK_DEV/lib/llvm/bin" \
+    HIP_DEVICE_LIB_PATH="$SDK_CORE/lib/llvm/amdgcn/bitcode"
+ENV LD_LIBRARY_PATH="$SDK_DEV/lib:$SDK_DEV/lib/rocm_sysdeps/lib:$SDK_CORE/lib:$SDK_CORE/lib/rocm_sysdeps/lib"
+ENV PATH="$SDK_DEV/bin:$SDK_DEV/lib/llvm/bin:$SDK_CORE/bin:$PATH"
 
 # Required for RCCL in ROCm7.1
 ENV HSA_NO_SCRATCH_RECLAIM=1
 
-ARG PYTHON_VERSION=3.12
+ARG PYTHON_VERSION
 ENV PYTHON_VERSION=${PYTHON_VERSION}
 
 RUN mkdir -p /app
 WORKDIR /app
 ENV DEBIAN_FRONTEND=noninteractive
 
-# Install Python and other dependencies (plus gcc-13, selected later in build_pytorch).
+# Install Python and other dependencies
 RUN apt-get update -y \
-    && apt-get install -y software-properties-common git curl sudo vim less libgfortran5 libopenmpi-dev libpci-dev liblzma-dev pkg-config \
+    && apt-get install -y software-properties-common build-essential git curl sudo vim less libgfortran5 libopenmpi-dev libpci-dev liblzma-dev pkg-config libdrm-dev libnuma-dev kmod \
     && for i in 1 2 3; do \
-        add-apt-repository -y ppa:deadsnakes/ppa && add-apt-repository -y ppa:ubuntu-toolchain-r/test && break || \
+        add-apt-repository -y ppa:deadsnakes/ppa && break || \
         { echo "Attempt $i failed, retrying in 5s..."; sleep 5; }; \
     done \
     && apt-get update -y \
     && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv \
-       python${PYTHON_VERSION}-lib2to3 python-is-python3 gcc-13 g++-13 \
+       python${PYTHON_VERSION}-lib2to3 python-is-python3 \
     && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \
     && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \
     && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \
     && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \
     && python3 --version && python3 -m pip --version
 
-RUN pip install -U packaging 'cmake<4' ninja wheel 'setuptools<80' 'pybind11<3.1.0' Cython
+RUN pip install -U packaging 'cmake<4' ninja wheel 'setuptools<80' pybind11==3.0.4 Cython
 RUN apt-get update && apt-get install -y libjpeg-dev libsox-dev libsox-fmt-all sox && rm -rf /var/lib/apt/lists/*
 
+# Install ROCm SDK via TheRock
+ARG ROCM_RELEASE_WHEELS_MULTIARCH_URL
+ARG ROCM_NIGHTLY_WHEELS_MULTIARCH_URL
+ARG ROCM_SDK_VERSION
+RUN pip install --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \
+    --extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \
+    "rocm[libraries,devel]${ROCM_SDK_VERSION:+==${ROCM_SDK_VERSION}}" \
+    && rocm-sdk init
+
+# Install PyTorch wheels (nightly for latest torch, release as fallback)
+ARG TORCH_VERSION
+ARG TORCHAUDIO_VERSION
+ARG TORCHVISION_VERSION
+RUN pip install \
+    --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \
+    --extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \
+    "torch==${TORCH_VERSION}" \
+    "torchaudio==${TORCHAUDIO_VERSION}" \
+    "torchvision==${TORCHVISION_VERSION}"
+
+# Install the AMD-compiled device kernels corresponding to PYTORCH_ROCM_ARCH:
+
+# Possible values that can be in PYTORCH_ROCM_ARCH:
+#
+# gfx90a (MI210/250), gfx942 (MI300), gfx950 (MI350), gfx1100 (7900XTX),
+# gfx1101 (7800XT), gfx1200 (9070XT), gfx1201 (9070), gfx1150/1151 (Strix APU)
+
+# For each arch in PYTORCH_ROCM_ARCH, install the precompiled GPU code that
+# PyTorch needs to run on that specific chip.
+SHELL ["/bin/bash", "-c"]
+RUN TORCH_VER=$(python3 -c "import torch; print(torch.__version__)") \
+    && for arch in $(echo "${PYTORCH_ROCM_ARCH}" | tr ';' ' '); do \
+        echo "Installing device packages for ${arch}" \
+        && pip install --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \
+            --extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \
+            "torch[device-${arch}]==${TORCH_VER}" \
+            "torchvision[device-${arch}]==${TORCHVISION_VERSION}" \
+            "rocm[device-${arch}]==${ROCM_SDK_VERSION}"; \
+    done
+
+# Remove gfx90a xnack kpacks (ASAN artifacts that cause hipErrorInvalidImage on MI250)
+RUN rm -f "${SITE_PACKAGES}/torch/.kpack/torch_gfx90a:xnack+.kpack" \
+         "${SITE_PACKAGES}/torch/.kpack/torch_gfx90a:xnack-.kpack"
+
+RUN cd $SITE_PACKAGES/_rocm_sdk_core/share/amd_smi && pip install .
+ENV PYTHONPATH=$SITE_PACKAGES/_rocm_sdk_core/share/amd_smi
+
+# TheRock runtime fixes
+RUN if [ -f "${SITE_PACKAGES}/rocm_sdk/__init__.py" ]; then \
+        sed -i 's/rtld_global: bool = True/rtld_global: bool = False/g' "${SITE_PACKAGES}/rocm_sdk/__init__.py"; \
+    fi \
+    && if [ -f "${SITE_PACKAGES}/_rocm_sdk_core/lib/librocm_smi64.so.1" ] \
+       && [ ! -e "${ROCM_PATH}/lib/librocm_smi64.so.7" ]; then \
+        ln -sfn "${SITE_PACKAGES}/_rocm_sdk_core/lib/librocm_smi64.so.1" "${ROCM_PATH}/lib/librocm_smi64.so.7"; \
+    fi \
+    && printf '%s\n%s\n' "${ROCM_PATH}/lib" "${SITE_PACKAGES}/_rocm_sdk_core/lib" > /etc/ld.so.conf.d/rocm-pip.conf \
+    && for f in ${SITE_PACKAGES}/_rocm_sdk_core/lib/libamdhip64.so*; do \
+        bn=$(basename "$f"); \
+        if [ -f "${ROCM_PATH}/lib/${bn}" ] && [ -f "$f" ] && [ ! -L "$f" ]; then \
+            rm -f "$f"; \
+            ln -sfn "${ROCM_PATH}/lib/${bn}" "$f"; \
+        fi; \
+    done \
+    && for f in librocprofiler-sdk.so.1 librocprofiler-register.so.0 \
+              librocprofiler-sdk-roctx.so.1; do \
+        if [ -e "${SITE_PACKAGES}/_rocm_sdk_core/lib/${f}" ] \
+           && [ -e "${SITE_PACKAGES}/_rocm_sdk_devel/lib/${f}" ]; then \
+            rm -f "${SITE_PACKAGES}/_rocm_sdk_core/lib/${f}"; \
+            ln -sfn "${SITE_PACKAGES}/_rocm_sdk_devel/lib/${f}" \
+                "${SITE_PACKAGES}/_rocm_sdk_core/lib/${f}"; \
+        fi; \
+    done \
+    && ldconfig
+
 # Install sccache if USE_SCCACHE is enabled (for release builds)
 ARG USE_SCCACHE
 ARG SCCACHE_DOWNLOAD_URL
@@ -94,141 +184,35 @@ RUN if [ "$USE_SCCACHE" = "1" ]; then \
 RUN if [ "$USE_SCCACHE" = "1" ]; then \
         echo "Setting up sccache wrappers for HIP compilation..." \
         && mkdir -p /opt/sccache-wrappers \
-        && printf '#!/bin/bash\nexec sccache /opt/rocm/lib/llvm/bin/clang++ "$@"\n' > /opt/sccache-wrappers/clang++ \
+        && printf '#!/bin/bash\nexec sccache %s/lib/llvm/bin/clang++ "$@"\n' "${ROCM_PATH}" > /opt/sccache-wrappers/clang++ \
         && chmod +x /opt/sccache-wrappers/clang++ \
-        && printf '#!/bin/bash\nexec sccache /opt/rocm/lib/llvm/bin/clang "$@"\n' > /opt/sccache-wrappers/clang \
+        && printf '#!/bin/bash\nexec sccache %s/lib/llvm/bin/clang "$@"\n' "${ROCM_PATH}" > /opt/sccache-wrappers/clang \
         && chmod +x /opt/sccache-wrappers/clang \
         && echo "sccache wrappers created in /opt/sccache-wrappers"; \
     fi
 
-# Set sccache environment variables only when USE_SCCACHE=1
-# This prevents S3 config from leaking into images when sccache is not used
-ARG USE_SCCACHE
 ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET_NAME}}
 ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}}
 ENV SCCACHE_S3_NO_CREDENTIALS=${USE_SCCACHE:+${SCCACHE_S3_NO_CREDENTIALS}}
 ENV SCCACHE_IDLE_TIMEOUT=${USE_SCCACHE:+0}
 
 
-###
-### Triton Build
-###
-FROM base AS build_triton
-ARG TRITON_BRANCH
-ARG TRITON_REPO
-RUN git clone ${TRITON_REPO}
-RUN cd triton \
-    && git checkout ${TRITON_BRANCH} \
-    && git config --global user.email "you@example.com" && git config --global user.name "Your Name" \
-    && if [ ! -f setup.py ]; then cd python; fi \
-    && MAX_JOBS=16 python3 setup.py bdist_wheel --dist-dir=dist \
-    && mkdir -p /app/install && cp dist/*.whl /app/install
-
-
-###
-### AMD SMI Build
-###
-FROM base AS build_amdsmi
-RUN cd /opt/rocm/share/amd_smi \
-    && pip wheel . --wheel-dir=dist
-RUN mkdir -p /app/install && cp /opt/rocm/share/amd_smi/dist/*.whl /app/install
-
-
-###
-### Pytorch build
-###
-FROM base AS build_pytorch
-ARG PYTORCH_BRANCH
-ARG PYTORCH_REPO
-ARG USE_SCCACHE
-
-RUN apt-get update && apt-get install -y pkg-config liblzma-dev
-# torch 2.12 emits float->__bf16 libcalls (__truncsfbf2) absent from gcc-11 libgcc; build with gcc-13.
-RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 --slave /usr/bin/g++ g++ /usr/bin/g++-13
-RUN git clone ${PYTORCH_REPO} pytorch
-RUN cd pytorch && git checkout ${PYTORCH_BRANCH}
-RUN cd pytorch \
-    && pip install -r requirements.txt && git submodule update --init --recursive
-RUN cd pytorch && python3 tools/amd_build/build_amd.py \
-    && if [ "$USE_SCCACHE" = "1" ]; then \
-           export HIP_CLANG_PATH=/opt/sccache-wrappers \
-           && export CMAKE_C_COMPILER_LAUNCHER=sccache \
-           && export CMAKE_CXX_COMPILER_LAUNCHER=sccache \
-           && sccache --show-stats; \
-       fi \
-    && CMAKE_PREFIX_PATH=$(python3 -c 'import sys; print(sys.prefix)') python3 setup.py bdist_wheel --dist-dir=dist \
-    && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
-RUN mkdir -p /app/install && cp /app/pytorch/dist/*.whl /app/install
-
-
-###
-### Shared PyTorch runtime for dependent builds
-###
-FROM base AS build_pytorch_runtime
-RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 --slave /usr/bin/g++ g++ /usr/bin/g++-13
-RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install,ro \
-    pip install /install/*.whl
-
-
-###
-### TorchVision build
-###
-FROM build_pytorch_runtime AS build_torchvision
-ARG PYTORCH_VISION_BRANCH
-ARG PYTORCH_VISION_REPO
-ARG USE_SCCACHE
-
-RUN git clone ${PYTORCH_VISION_REPO} vision
-RUN cd vision && git checkout ${PYTORCH_VISION_BRANCH} \
-    && if [ "$USE_SCCACHE" = "1" ]; then \
-           export HIP_CLANG_PATH=/opt/sccache-wrappers \
-           && export CMAKE_C_COMPILER_LAUNCHER=sccache \
-           && export CMAKE_CXX_COMPILER_LAUNCHER=sccache; \
-       fi \
-    && python3 setup.py bdist_wheel --dist-dir=dist \
-    && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
-RUN mkdir -p /app/install && cp /app/vision/dist/*.whl /app/install
-
-
-###
-### TorchAudio build
-###
-FROM build_pytorch_runtime AS build_torchaudio
-ARG PYTORCH_AUDIO_BRANCH
-ARG PYTORCH_AUDIO_REPO
-ARG USE_SCCACHE
-
-RUN git clone ${PYTORCH_AUDIO_REPO} audio
-RUN cd audio && git checkout ${PYTORCH_AUDIO_BRANCH} \
-    && git submodule update --init --recursive
-RUN cd audio && pip install -r requirements.txt
-RUN cd audio \
-    && if [ "$USE_SCCACHE" = "1" ]; then \
-           export HIP_CLANG_PATH=/opt/sccache-wrappers \
-           && export CMAKE_C_COMPILER_LAUNCHER=sccache \
-           && export CMAKE_CXX_COMPILER_LAUNCHER=sccache; \
-       fi \
-    && python3 setup.py bdist_wheel --dist-dir=dist \
-    && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi
-RUN mkdir -p /app/install && cp /app/audio/dist/*.whl /app/install
-
-
 ###
 ### MORI Build
 ###
 FROM base AS build_mori
 ARG MORI_BRANCH
 ARG MORI_REPO
-RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \
-    pip install /install/*.whl
 RUN git clone ${MORI_REPO}
 RUN cd mori \
     && git checkout ${MORI_BRANCH} \
     && git submodule update --init --recursive \
-    && python3 setup.py bdist_wheel --dist-dir=dist && ls /app/mori/dist/*.whl
+    && ln -sfn ${ROCM_PATH} /opt/rocm \
+    && pip install "setuptools_scm>=6.2" \
+    && CMAKE_PREFIX_PATH="${ROCM_PATH}:${ROCM_PATH}/lib/rocm_sysdeps" \
+       python3 setup.py bdist_wheel --dist-dir=dist && ls /app/mori/dist/*.whl
 RUN mkdir -p /app/install && cp /app/mori/dist/*.whl /app/install
 
-
 ###
 ### FlashAttention Build
 ###
@@ -236,8 +220,6 @@ FROM base AS build_fa
 ARG FA_BRANCH
 ARG FA_REPO
 ARG USE_SCCACHE
-RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \
-    pip install /install/*.whl
 RUN git clone ${FA_REPO}
 RUN cd flash-attention \
     && git checkout ${FA_BRANCH} \
@@ -258,12 +240,12 @@ FROM base AS build_aiter
 ARG AITER_BRANCH
 ARG AITER_REPO
 ARG USE_SCCACHE
-RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \
-    pip install /install/*.whl
 RUN git clone --recursive --branch ${AITER_BRANCH} ${AITER_REPO}
 RUN cd aiter \
     && git submodule update --init --recursive \
     && pip install -r requirements.txt
+# AITER_USE_SYSTEM_TRITON keeps setup.py from resolving its own triton, which would
+# displace the pinned one already installed in base.
 RUN pip install pyyaml && cd aiter \
     && if [ "$USE_SCCACHE" = "1" ]; then \
            export HIP_CLANG_PATH=/opt/sccache-wrappers \
@@ -276,58 +258,30 @@ RUN mkdir -p /app/install && cp /app/aiter/dist/*.whl /app/install
 
 
 ###
-### Final Build
+### Triton Build
 ###
+FROM base AS build_triton
+ARG TRITON_BRANCH
+ARG TRITON_REPO
+RUN git clone ${TRITON_REPO}
+RUN cd triton \
+    && git checkout ${TRITON_BRANCH} \
+    && if [ ! -f setup.py ]; then cd python; fi \
+    && python3 setup.py bdist_wheel --dist-dir=dist \
+    && mkdir -p /app/install && cp dist/*.whl /app/install
 
-# Wheel release stage - 
-# only includes dependencies used by wheel release pipeline
-FROM base AS debs_wheel_release
-RUN mkdir /app/debs
-RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-RUN --mount=type=bind,from=build_amdsmi,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-RUN --mount=type=bind,from=build_torchvision,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-RUN --mount=type=bind,from=build_torchaudio,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-
-# Full debs stage - includes Mori (used by Docker releases)
-FROM base AS debs
-RUN mkdir /app/debs
-RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-RUN --mount=type=bind,from=build_amdsmi,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-RUN --mount=type=bind,from=build_torchvision,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-RUN --mount=type=bind,from=build_torchaudio,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
-RUN --mount=type=bind,from=build_mori,src=/app/install/,target=/install \
-    cp /install/*.whl /app/debs
 
 ###
 ### ROCr + CLR build
 ###
-# Pull in updates to ROCr + clr from ROCm 7.2.4 and beyond.
+# Brings in multi-stream perf improvements; drop after these arrive in an official release
 FROM base AS build_rocm_runtime
 ARG ROCM_SYSTEMS_REPO
 ARG ROCM_RUNTIME_COMMIT
+# libgl-dev: rocclr does find_package(OpenGL REQUIRED) unconditionally and the wheel image has
+# no GL headers. The link must not pick up a libGL DT_NEEDED, which is asserted after the build.
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends g++ \
+    && apt-get install -y --no-install-recommends g++ libdw-dev xxd libgl-dev libglx-dev \
-       libelf-dev libdrm-dev libnuma-dev libdw-dev xxd \
     && rm -rf /var/lib/apt/lists/*
 RUN pip install --no-cache-dir CppHeaderParser
 RUN git init -q /src && cd /src \
@@ -339,20 +293,17 @@ RUN git init -q /src && cd /src \
     && git sparse-checkout set projects/rocr-runtime projects/clr projects/hip shared cmake \
     && git fetch --filter=blob:none --depth 1 origin ${ROCM_RUNTIME_COMMIT} \
     && git checkout -q FETCH_HEAD
-# ROCr's trap_handler and blit_shaders call find_package(Clang/LLVM REQUIRED) only to get
+# libelf is taken from the SDK's own vendored sysdeps rather than apt -- the wheel image has no
-# IMPORTED executables. The ROCm dev image ships the binaries but no CMake package config, so
+# /usr/include/libelf.h -- so ROCr links against the copy the wheel itself was built against.
-# supply one pointing at them.
+ENV SYSDEPS="$SDK_DEV/lib/rocm_sysdeps"
-RUN mkdir -p /opt/rocm-cmake-shim \
+# ROCr first, and installed into $SDK_DEV, because the clr build below needs its headers.
-    && printf 'if(NOT TARGET clang)\n  add_executable(clang IMPORTED GLOBAL)\n  set_target_properties(clang PROPERTIES IMPORTED_LOCATION "/opt/rocm/llvm/bin/clang")\nendif()\nset(Clang_PACKAGE_VERSION "rocm-image-shim")\n' > /opt/rocm-cmake-shim/ClangConfig.cmake \
-    && printf 'if(NOT TARGET llvm-objcopy)\n  add_executable(llvm-objcopy IMPORTED GLOBAL)\n  set_target_properties(llvm-objcopy PROPERTIES IMPORTED_LOCATION "/opt/rocm/llvm/bin/llvm-objcopy")\nendif()\nset(LLVM_FOUND TRUE)\nset(LLVM_PACKAGE_VERSION "rocm-image-shim")\n' > /opt/rocm-cmake-shim/LLVMConfig.cmake
-# ROCr first, and installed into /opt/rocm, because the clr build below needs its headers
 RUN cd /src/projects/rocr-runtime \
     && cmake -B build -G Ninja \
         -DCMAKE_BUILD_TYPE=Release \
-        -DCMAKE_PREFIX_PATH=/opt/rocm \
+        -DCMAKE_PREFIX_PATH="$SDK_DEV;$SYSDEPS" \
-        -DCMAKE_INSTALL_PREFIX=/opt/rocm \
+        -DCMAKE_INSTALL_PREFIX="$SDK_DEV" \
-        -DClang_DIR=/opt/rocm-cmake-shim \
+        -DLIBELF_INCLUDE_DIRS="$SYSDEPS/include" \
-        -DLLVM_DIR=/opt/rocm-cmake-shim \
+        -DLIBELF_LIBRARIES="$SYSDEPS/lib/libelf.so" \
     && cmake --build build --parallel 16 \
     && cmake --install build \
     && cmake --install build --prefix /rocr-install --strip
@@ -361,12 +312,12 @@ RUN cd /src/projects/clr \
         -DCMAKE_BUILD_TYPE=Release \
         -DCLR_BUILD_HIP=ON \
         -DCLR_BUILD_OCL=OFF \
+        -DROCM_KPACK_ENABLED=ON \
         -DHIP_COMMON_DIR=/src/projects/hip \
-        -DROCM_PATH=/opt/rocm \
+        -DROCM_PATH="$SDK_DEV" \
-        -DCMAKE_PREFIX_PATH=/opt/rocm \
+        -DCMAKE_PREFIX_PATH="$SDK_DEV;$SYSDEPS" \
-        -DCMAKE_INSTALL_PREFIX=/opt/rocm \
+        -DCMAKE_INSTALL_PREFIX="$SDK_DEV" \
-        -DLLVM_DIR=/opt/rocm-cmake-shim \
+        -DHIP_LLVM_ROOT="$SDK_DEV/lib/llvm" \
-        -DHIP_LLVM_ROOT=/opt/rocm/llvm \
     && cmake --build build --parallel 16 \
     && cmake --install build --prefix /clr-install --strip
 RUN mkdir -p /staging/lib \
@@ -374,82 +325,115 @@ RUN mkdir -p /staging/lib \
     && cp -P /clr-install/lib/libamdhip64.so* /staging/lib/ \
     && ls -l /staging/lib
 
-# rocprofiler-sdk 1.3.2 fixes torch 2.12 profiler on ROCm 7.2.3
+
-FROM base AS build_rocprofiler_sdk
+###
-ARG ROCM_SYSTEMS_REPO
+### Final Build
-ARG ROCPROFILER_SDK_COMMIT
+###
-ARG USE_SCCACHE
+
-RUN apt-get update \
+# Wheel release stage -
-    && apt-get install -y --no-install-recommends libdw-dev libelf-dev libsqlite3-dev libzstd-dev \
+# only includes dependencies used by wheel release pipeline
-    && rm -rf /var/lib/apt/lists/*
+FROM base AS debs_wheel_release
-RUN git init -q /src && cd /src \
+RUN mkdir /app/debs
-    && git remote add origin ${ROCM_SYSTEMS_REPO} \
+RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \
-    && git config core.sparseCheckout true \
+    cp /install/*.whl /app/debs
-    && git config remote.origin.promisor true \
+RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \
-    && git config remote.origin.partialclonefilter blob:none \
+    cp /install/*.whl /app/debs
-    && git sparse-checkout init --cone \
+RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
-    && git sparse-checkout set projects/rocprofiler-sdk shared cmake \
+    cp /install/*.whl /app/debs
-    && git fetch --filter=blob:none --depth 1 origin ${ROCPROFILER_SDK_COMMIT} \
+# The wheel index also serves a torch repacked to require our triton instead of
-    && git checkout -q FETCH_HEAD
+# the one TheRock built it with, and amdsmi, which TheRock ships only as sources
-# Patch the ROCm 7.2.3 build of rocprofiler-sdk to include queue-interposition fixes that are
+# inside rocm-sdk-core.
-# absent from the pinned 7.14 build. This resolves hangs on torch.profiler on ROCm.
+ARG TORCH_VERSION
-# TODO: Drop these once ${ROCPROFILER_SDK_COMMIT} is bumped to a build that already 
+ARG ROCM_RELEASE_WHEELS_MULTIARCH_URL
-# contains them, such as ROCm 10.0. See https://github.com/vllm-project/vllm/issues/54087.
+ARG ROCM_NIGHTLY_WHEELS_MULTIARCH_URL
-ARG ROCPROFILER_SDK_PR_7796=fae7fd57ffb8a76e811d6410b8a0cee1e94d710a
+COPY tools/vllm-rocm/therock_wheels.py /tmp/therock_wheels.py
-ARG ROCPROFILER_SDK_PR_7924=f8db22adbe17686a9b05ae03367dd1ec73011653
+# regex and pybase64 aren't installed here; fall back to the stdlib modules.
-RUN cd /src \
+RUN sed -i -e 's/import regex as re/import re/' \
-    && HSA_PATHS='projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/*' \
+        -e 's/import pybase64 as base64/import base64/' /tmp/therock_wheels.py \
-    && curl -fsSL -o /tmp/qi-7796.patch "${ROCM_SYSTEMS_REPO%.git}/commit/${ROCPROFILER_SDK_PR_7796}.patch" \
+    && pip download --no-deps --dest /tmp/torch-dl \
-    && curl -fsSL -o /tmp/qi-7924.patch "${ROCM_SYSTEMS_REPO%.git}/commit/${ROCPROFILER_SDK_PR_7924}.patch" \
+        --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \
-    && git apply -p1 --include="$HSA_PATHS" --verbose /tmp/qi-7796.patch \
+        --extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \
-    && git apply -p1 --include="$HSA_PATHS" --verbose /tmp/qi-7924.patch
+        "torch==${TORCH_VERSION}" \
-RUN cd /src/projects/rocprofiler-sdk \
+    && TRITON_VER=$(basename /app/debs/triton-*.whl | cut -d- -f2) \
-    && if [ "$USE_SCCACHE" = "1" ]; then \
+    && python3 /tmp/therock_wheels.py repack-torch /tmp/torch-dl/torch-*.whl \
-           export CMAKE_C_COMPILER_LAUNCHER=sccache \
+        --triton-version "${TRITON_VER}" --out-dir /app/debs \
-           && export CMAKE_CXX_COMPILER_LAUNCHER=sccache; \
+    && python3 /tmp/therock_wheels.py patch-amdsmi "${SDK_CORE}/share/amd_smi" \
-       fi \
+        --out-dir /tmp/amd_smi \
-    && cmake -B build -G Ninja \
+    && pip wheel --no-deps --wheel-dir /app/debs /tmp/amd_smi \
-        -DCMAKE_BUILD_TYPE=Release \
+    && rm -rf /tmp/torch-dl /tmp/amd_smi /tmp/therock_wheels.py \
-        -DCMAKE_PREFIX_PATH=/opt/rocm \
+    && ls -l /app/debs
-        -DCMAKE_INSTALL_PREFIX=/opt/rocm \
+
-        -DROCPROFILER_BUILD_TESTS=OFF \
+FROM base AS debs
-        -DROCPROFILER_BUILD_SAMPLES=OFF \
+RUN mkdir /app/debs
-        -DROCPROFILER_BUILD_DOCS=OFF \
+RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \
-    && cmake --build build --parallel "$(nproc)" \
+    cp /install/*.whl /app/debs
-    && cmake --install build --prefix /staging --strip
+RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \
+    cp /install/*.whl /app/debs
+RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \
+    cp /install/*.whl /app/debs
+RUN --mount=type=bind,from=build_mori,src=/app/install/,target=/install \
+    cp /install/*.whl /app/debs
 
 FROM base AS final
-RUN --mount=type=bind,from=debs,src=/app/debs,target=/install \
+ARG TRITON_VERSION
-    pip install /install/*.whl
 
-COPY --from=build_rocprofiler_sdk /staging/ /opt/rocm/
+# torch 2.12 implicitly attaches rocprofiler-sdk; its queue interposition hangs at HSA
+# teardown, so disable it. TODO: drop once ROCm's stock rocprofiler carries the fix.
 ENV ROCPROFILER_QUEUE_INTERPOSITION=0
 
-COPY --from=build_rocm_runtime /staging/ /opt/rocm/
+COPY --from=build_rocm_runtime /staging/lib/ ${SDK_DEV}/lib/
-# Hard link the patched library onto every stock file name so that both the
+# Hard link the patched library onto every stock file name, in both SDK trees, so that the
-# SONAME lookup and the hard coded paths resolve to it.
+# SONAME lookup and the hard coded paths all resolve to it. Hard links rather than symlinks
+# because ldconfig rewrites SONAME symlinks but cannot re-point an inode. $SDK_CORE keeps its
+# own libhsa-runtime64 copy, which is on LD_LIBRARY_PATH and in ld.so.conf; relinking it is
+# belt and braces rather than strictly required, but it leaves no stock ROCr in the image.
 RUN set -eux; \
-    cd /opt/rocm/lib; \
+    cd "$SDK_DEV/lib"; \
     ours_hsa="$(readlink -f libhsa-runtime64.so.1)"; \
     ours_hip="$(readlink -f libamdhip64.so.7)"; \
-    for f in libhsa-runtime64.so.1.*; do \
+    for f in "$SDK_DEV"/lib/libhsa-runtime64.so.1* "$SDK_CORE"/lib/libhsa-runtime64.so.1* \
-        [ "$f" = "${ours_hsa##*/}" ] || ln -f "$ours_hsa" "$f"; \
+             "$SDK_DEV"/lib/libamdhip64.so.7* "$SDK_CORE"/lib/libamdhip64.so.7*; do \
-    done; \
+        [ -f "$f" ] && [ ! -L "$f" ] || continue; \
-    for f in libamdhip64.so.7.*; do \
+        case "${f##*/}" in libhsa*) ours="$ours_hsa";; *) ours="$ours_hip";; esac; \
-        [ "$f" = "${ours_hip##*/}" ] || ln -f "$ours_hip" "$f"; \
+        [ "$f" -ef "$ours" ] || ln -f "$ours" "$f"; \
     done
 
 RUN ldconfig
 
+# This is to prevent resolvers from checking METADATA and RECORD and clobbering
+# the Triton source build we installed. Constraints files are unable to help
+# with this. However, these entries are optional and can be blanked which
+# solves the problem.
+RUN TORCH_DIST_INFO=$(ls -d ${SITE_PACKAGES}/torch-*.dist-info) \
+    && grep -q '^Requires-Dist: triton' "$TORCH_DIST_INFO/METADATA" \
+    && sed -i '/^Requires-Dist: triton/d' "$TORCH_DIST_INFO/METADATA" \
+    && sed -i "s|^\(${TORCH_DIST_INFO##*/}/METADATA\),.*|\1,,|" "$TORCH_DIST_INFO/RECORD"
+
+RUN --mount=type=bind,from=debs,src=/app/debs,target=/install \
+    pip install /install/*.whl \
+    && ! pip check 2>&1 | grep -iE '^(torch|triton) ' \
+    && test "$(pip show triton | awk '/^Version:/ {print $2}')" = "${TRITON_VERSION}" \
+    # Create a constraints file to protect the versions we need, especially
+    # triton, requirements will try to reinstall repeatedly.
+    && pip list --format=freeze \
+        | grep -E '^(torch|triton|torchaudio|torchvision|pybind11)==' > /etc/rocm-constraints.txt \
+    && echo "ROCm constraints:" && cat /etc/rocm-constraints.txt \
+    && test "$(wc -l < /etc/rocm-constraints.txt)" -eq 5 \
+    && grep -q '^pybind11==3\.0\.' /etc/rocm-constraints.txt \
+    && python3 -c "import torch; assert torch.version.hip, f'non-ROCm torch: {torch.__version__}'; print(f'torch={torch.__version__}')" \
+    && python3 -c "import triton; print(f'triton={triton.__version__}')" \
+    && pip list 2>/dev/null | grep -iE '^(torch|triton) '
+
 ARG BASE_IMAGE
+ARG PYTHON_VERSION
+ARG ROCM_SDK_VERSION
+ARG TORCH_VERSION
+ARG TORCHAUDIO_VERSION
+ARG TORCHVISION_VERSION
+ARG PYTORCH_ROCM_ARCH
+ARG AITER_ROCM_ARCH
 ARG TRITON_BRANCH
 ARG TRITON_REPO
-ARG PYTORCH_BRANCH
-ARG PYTORCH_VISION_BRANCH
-ARG PYTORCH_REPO
-ARG PYTORCH_VISION_REPO
-ARG PYTORCH_AUDIO_BRANCH
-ARG PYTORCH_AUDIO_REPO
 ARG FA_BRANCH
 ARG FA_REPO
 ARG AITER_BRANCH
@@ -457,23 +441,36 @@ ARG AITER_REPO
 ARG MORI_BRANCH
 ARG MORI_REPO
 ARG ROCM_SYSTEMS_REPO
-ARG ROCPROFILER_SDK_COMMIT
 ARG ROCM_RUNTIME_COMMIT
+ARG TRITON_VERSION
+# Record hashes for Triton and pybind11 so we can verify later.
 RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \
-    && echo "ROCM_SYSTEMS_REPO: ${ROCM_SYSTEMS_REPO}" >> /app/versions.txt \
+    && echo "PYTHON_VERSION: ${PYTHON_VERSION}" >> /app/versions.txt \
-    && echo "ROCPROFILER_SDK_COMMIT: ${ROCPROFILER_SDK_COMMIT}" >> /app/versions.txt \
+    && echo "ROCM_SDK_VERSION: ${ROCM_SDK_VERSION}" >> /app/versions.txt \
-    && echo "ROCM_RUNTIME_COMMIT: ${ROCM_RUNTIME_COMMIT}" >> /app/versions.txt \
+    && echo "TORCH_VERSION: ${TORCH_VERSION}" >> /app/versions.txt \
+    && echo "TORCHAUDIO_VERSION: ${TORCHAUDIO_VERSION}" >> /app/versions.txt \
+    && echo "TORCHVISION_VERSION: ${TORCHVISION_VERSION}" >> /app/versions.txt \
+    && echo "PYTORCH_ROCM_ARCH: ${PYTORCH_ROCM_ARCH}" >> /app/versions.txt \
+    && echo "AITER_ROCM_ARCH: ${AITER_ROCM_ARCH}" >> /app/versions.txt \
     && echo "TRITON_BRANCH: ${TRITON_BRANCH}" >> /app/versions.txt \
     && echo "TRITON_REPO: ${TRITON_REPO}" >> /app/versions.txt \
-    && echo "PYTORCH_BRANCH: ${PYTORCH_BRANCH}" >> /app/versions.txt \
-    && echo "PYTORCH_VISION_BRANCH: ${PYTORCH_VISION_BRANCH}" >> /app/versions.txt \
-    && echo "PYTORCH_REPO: ${PYTORCH_REPO}" >> /app/versions.txt \
-    && echo "PYTORCH_VISION_REPO: ${PYTORCH_VISION_REPO}" >> /app/versions.txt \
-    && echo "PYTORCH_AUDIO_BRANCH: ${PYTORCH_AUDIO_BRANCH}" >> /app/versions.txt \
-    && echo "PYTORCH_AUDIO_REPO: ${PYTORCH_AUDIO_REPO}" >> /app/versions.txt \
     && echo "FA_BRANCH: ${FA_BRANCH}" >> /app/versions.txt \
     && echo "FA_REPO: ${FA_REPO}" >> /app/versions.txt \
     && echo "AITER_BRANCH: ${AITER_BRANCH}" >> /app/versions.txt \
     && echo "AITER_REPO: ${AITER_REPO}" >> /app/versions.txt \
     && echo "MORI_BRANCH: ${MORI_BRANCH}" >> /app/versions.txt \
-    && echo "MORI_REPO: ${MORI_REPO}" >> /app/versions.txt
+    && echo "MORI_REPO: ${MORI_REPO}" >> /app/versions.txt \
+    && echo "ROCM_SYSTEMS_REPO: ${ROCM_SYSTEMS_REPO}" >> /app/versions.txt \
+    && echo "ROCM_RUNTIME_COMMIT: ${ROCM_RUNTIME_COMMIT}" >> /app/versions.txt \
+    # Fingerprint our triton so Dockerfile.rocm can prove it survived the build.
+    && echo "TRITON_VERSION: $(pip show triton | awk '/^Version:/ {print $2}')" >> /app/versions.txt \
+    && echo "TRITON_SHA256: $(find $(python3 -c 'import triton, os; print(os.path.dirname(triton.__file__))') \
+        -type f \( -name '*.py' -o -name '*.so' \) | sort | xargs sha256sum | sha256sum | cut -d' ' -f1)" \
+        >> /app/versions.txt \
+    && echo "PYBIND11_VERSION: $(pip show pybind11 | awk '/^Version:/ {print $2}')" >> /app/versions.txt \
+    && echo "PYBIND11_INTERNALS_VERSION: $(grep -rhE '^#[[:space:]]*define PYBIND11_INTERNALS_VERSION' \
+        $(python3 -c 'import pybind11; print(pybind11.get_include())') | awk '{print $NF}')" \
+        >> /app/versions.txt \
+    && echo "PYBIND11_SHA256: $(find $(python3 -c 'import pybind11; print(pybind11.get_include())') \
+        -type f -name '*.h' | sort | xargs sha256sum | sha256sum | cut -d' ' -f1)" \
+        >> /app/versions.txt

docker/ci-rocm.hcl

@@ -83,10 +83,6 @@ variable "UCX_BRANCH" {
   default = ""
 }
 
-variable "ROCSHMEM_BRANCH" {
-  default = ""
-}
-
 variable "DEEPEP_BRANCH" {
   default = ""
 }
@@ -95,10 +91,6 @@ variable "NIXL_CACHE_KEY" {
   default = ""
 }
 
-variable "ROCSHMEM_CACHE_KEY" {
-  default = ""
-}
-
 variable "DEEPEP_CACHE_KEY" {
   default = ""
 }
@@ -236,7 +228,7 @@ function "get_cache_to_rocm_rust" {
   ])
 }
 
-# Cache functions for upstream dependency stages (NIXL/UCX, ROCShmem, DeepEP).
+# Cache functions for upstream dependency stages (NIXL/UCX, DeepEP).
 # These stages are pinned to specific upstream commit hashes, so cache keys use
 # those hashes rather than the Buildkite commit. This means the cache persists
 # across all vLLM commits as long as the upstream dependency pins don't change.
@@ -245,8 +237,7 @@ function "get_cache_from_rocm_deps" {
   params = []
   result = compact([
     NIXL_CACHE_KEY != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:nixl-rocm-${NIXL_CACHE_KEY}" : (NIXL_BRANCH != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:nixl-rocm-${NIXL_BRANCH}-ucx-${UCX_BRANCH}" : ""),
-    ROCSHMEM_CACHE_KEY != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rocshmem-rocm-${ROCSHMEM_CACHE_KEY}" : (ROCSHMEM_BRANCH != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rocshmem-rocm-${ROCSHMEM_BRANCH}" : ""),
+    DEEPEP_CACHE_KEY != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:deepep-rocm-${DEEPEP_CACHE_KEY}" : (DEEPEP_BRANCH != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:deepep-rocm-${DEEPEP_BRANCH}" : ""),
-    DEEPEP_CACHE_KEY != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:deepep-rocm-${DEEPEP_CACHE_KEY}" : (DEEPEP_BRANCH != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:deepep-rocm-${DEEPEP_BRANCH}-rocshmem-${ROCSHMEM_BRANCH}" : ""),
   ])
 }
 
@@ -257,17 +248,10 @@ function "get_cache_to_rocm_nixl" {
   ])
 }
 
-function "get_cache_to_rocm_rocshmem" {
-  params = []
-  result = compact([
-    ROCSHMEM_CACHE_KEY != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rocshmem-rocm-${ROCSHMEM_CACHE_KEY},mode=min" : (ROCSHMEM_BRANCH != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rocshmem-rocm-${ROCSHMEM_BRANCH},mode=min" : ""),
-  ])
-}
-
 function "get_cache_to_rocm_deepep" {
   params = []
   result = compact([
-    DEEPEP_CACHE_KEY != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:deepep-rocm-${DEEPEP_CACHE_KEY},mode=min" : (DEEPEP_BRANCH != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:deepep-rocm-${DEEPEP_BRANCH}-rocshmem-${ROCSHMEM_BRANCH},mode=min" : ""),
+    DEEPEP_CACHE_KEY != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:deepep-rocm-${DEEPEP_CACHE_KEY},mode=min" : (DEEPEP_BRANCH != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:deepep-rocm-${DEEPEP_BRANCH},mode=min" : ""),
   ])
 }
 
@@ -385,14 +369,6 @@ target "nixl-rocm-ci" {
   output     = ["type=cacheonly"]
 }
 
-target "rocshmem-rocm-ci" {
-  inherits   = ["_common-rocm", "_ci-rocm"]
-  target     = "build_rocshmem"
-  cache-from = get_cache_from_rocm_deps()
-  cache-to   = get_cache_to_rocm_rocshmem()
-  output     = ["type=cacheonly"]
-}
-
 target "deepep-rocm-ci" {
   inherits   = ["_common-rocm", "_ci-rocm"]
   target     = "build_deepep"
@@ -418,7 +394,7 @@ target "ci-base-rocm-ci" {
       CI_BASE_IMAGE_TAG != "" ? "type=registry,ref=${CI_BASE_IMAGE_TAG}" : "",
       CI_BASE_TRUSTED_CONTENT_REF != "" ? "type=registry,ref=${CI_BASE_TRUSTED_CONTENT_REF}" : "",
     ]),
-    # Import upstream dependency caches so NIXL/ROCShmem/DeepEP stages
+    # Import upstream dependency caches so NIXL/DeepEP stages
     # are cache hits even when ci_base itself needs rebuilding.
     get_cache_from_rocm_deps(),
   )
@@ -431,5 +407,5 @@ target "ci-base-rocm-ci" {
 # Group for ci_base builds -- exports dependency stage caches alongside the
 # ci_base image so future rebuilds can reuse them independently.
 group "ci-base-rocm-ci-with-deps" {
-  targets = ["nixl-rocm-ci", "rocshmem-rocm-ci", "deepep-rocm-ci", "ci-base-rocm-ci"]
+  targets = ["nixl-rocm-ci", "deepep-rocm-ci", "ci-base-rocm-ci"]
 }

docker/docker-bake-rocm.hcl

@@ -65,10 +65,6 @@ variable "UCX_BRANCH" {
   default = ""
 }
 
-variable "ROCSHMEM_BRANCH" {
-  default = ""
-}
-
 variable "DEEPEP_BRANCH" {
   default = ""
 }

docs/getting_started/installation/gpu.rocm.inc.md

@@ -1,7 +1,7 @@
 <!-- markdownlint-disable MD041 MD051 -->
 --8<-- [start:installation]
 
-vLLM supports AMD GPUs with ROCm 6.3 or above. Pre-built wheels are available for ROCm 7.0 and ROCm 7.2.1.
+vLLM supports AMD GPUs with ROCm 6.3 or above. Pre-built wheels are available for ROCm 10.0 (the default) and ROCm 7.2.
 
 #### Prebuilt Wheels
 
@@ -9,6 +9,8 @@ vLLM supports AMD GPUs with ROCm 6.3 or above. Pre-built wheels are available fo
 | ------------ | -------------- | ------------ | ----------------- | ------------------ |
 | `rocm700` | 3.12 | 7.0 | >= 2.35 | `0.14.0` to `0.18.0` |
 | `rocm721` | 3.12 | 7.2.1 | >= 2.35 | Nightly releases after commit `171775f306a333a9cf105bfd533bf3e113d401d9` |
+| `rocm72` | 3.12 | 7.2 | >= 2.39 | Nightly releases on the legacy ROCm 7.2 stack |
+| `rocm100` | 3.12 | 10.0 | >= 2.35 | Nightly releases (default) |
 
 --8<-- [end:installation]
 --8<-- [start:requirements]
@@ -40,12 +42,17 @@ If you need a different ROCm version or want to use an existing PyTorch installa
     source .venv/bin/activate
     ```
 
-To install the latest version of vLLM for Python 3.12, ROCm 7.0 and `glibc >= 2.35`.
+To install the latest version of vLLM for Python 3.12, ROCm 10.0 and `glibc >= 2.35`:
 
 ```bash
 uv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm/ --upgrade
 ```
 
+!!! tip
+    The ROCm 10.0 wheels install the ROCm SDK from pip ([TheRock](https://github.com/ROCm/TheRock)), including the GPU kernels for every supported architecture.
+
+    Wheels for the legacy ROCm 7.2 stack, which use a system ROCm installation, remain available at `https://wheels.vllm.ai/rocm/${VLLM_VERSION}/rocm72`.
+
 !!! tip
     You can find out about which ROCm version the latest vLLM supports by checking the `vllm` package in index in extra-index-url <https://wheels.vllm.ai/rocm/> at [https://wheels.vllm.ai/rocm/vllm](https://wheels.vllm.ai/rocm/vllm) .
 
@@ -195,9 +202,9 @@ install it without re-running the full installation:
     and place it in `vllm/vllm-rs`.
 
 If you see an error about a wheel not being found, the wheel for your base
-commit and ROCm patch version might not be available. Check the available
+commit and ROCm version might not be available. Check the available
-variants under `https://wheels.vllm.ai/rocm/<commit>/`. For example, ROCm 7.2.1
+variants under `https://wheels.vllm.ai/rocm/<commit>/`. Variants are named by ROCm
-uses the `rocm721` variant.
+major and minor version, e.g. ROCm 7.2.3 uses the `rocm72` variant.
 
 There are more environment variables to control the behavior of Python-only
 build:
@@ -208,7 +215,7 @@ build:
 - `VLLM_PRECOMPILED_WHEEL_COMMIT`: override the full commit hash used to
   download the pre-compiled wheel.
 - `VLLM_PRECOMPILED_WHEEL_VARIANT`: specify the ROCm variant subdirectory, e.g.,
-  `rocm700` or `rocm721`. If not specified, the variant is auto-detected based
+  `rocm72` or `rocm100`. If not specified, the variant is auto-detected based
   on your system's ROCm version. An explicitly specified variant must match the
   detected environment.
 
@@ -226,23 +233,34 @@ You can find more information about vLLM's wheels in
 #### Full build (with compilation) {#full-build}
 
 !!! tip
-    - If you found that the following installation step does not work for you, please refer to [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base). Dockerfile is a form of installation steps.
+    - The steps below follow [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base), which pins the validated version of every component. If a step does not work for you, refer to it; a Dockerfile is a form of installation steps.
+    - The legacy ROCm 7.2 stack uses a system ROCm installation instead; its steps are in [docker/Dockerfile.rocm_72_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_72_base).
 
-0. Install prerequisites (skip if you are already in an environment/docker with the following installed):
+0. Install the ROCm SDK and PyTorch (skip if you are already in an environment or docker image with them installed, e.g. `vllm/vllm-openai-rocm:base-nightly`).
 
-    - [ROCm](https://rocm.docs.amd.com/en/latest/deploy/linux/index.html)
+    ROCm 10.0 is installed from pip ([TheRock](https://github.com/ROCm/TheRock)), together with PyTorch and the GPU kernels for your GPU's architecture. In a Python 3.12 environment:
-    - [PyTorch](https://pytorch.org/)
-
-    For installing PyTorch, you can start from a fresh docker image, e.g, `rocm/pytorch:rocm7.0_ubuntu22.04_py3.10_pytorch_release_2.8.0`, `rocm/pytorch-nightly`. If you are using docker image, you can skip to Step 3.
-
-    Alternatively, you can install PyTorch using PyTorch wheels. You can check PyTorch installation guide in PyTorch [Getting Started](https://pytorch.org/get-started/locally/). Example:
 
     ```bash
-    # Install PyTorch
+    # Your GPU's architecture, e.g. gfx942 for MI300 or gfx950 for MI350
-    pip uninstall torch -y
+    export GPU_ARCH=$(uvx --from rocm-bootstrap rocm-bootstrap-detect --unique)
-    pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm7.0
+    export ROCM_INDEX=https://stable.repo.amd.com/rocm/whl-next/
+
+    pip install --index-url ${ROCM_INDEX} \
+        "rocm[libraries,devel,device-${GPU_ARCH}]==10.0.0" \
+        "torch[device-${GPU_ARCH}]==2.12.0+rocm10.0.0" \
+        "torchvision[device-${GPU_ARCH}]==0.27.0+rocm10.0.0"
+    rocm-sdk init
+
+    # Point builds at the pip-installed ROCm SDK
+    export ROCM_PATH=$(rocm-sdk path --root)
+    export ROCM_HOME=${ROCM_PATH} HIP_PATH=${ROCM_PATH}
+    export PATH=$(rocm-sdk path --bin):${PATH}
     ```
 
+    !!! note
+        - The validated `ROCM_SDK_VERSION`, `TORCH_VERSION` and `TORCHVISION_VERSION` can be found in [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base).
+        - To support several GPU architectures, add a `device-<arch>` extra for each of them.
+
 1. Install [Triton for ROCm](https://github.com/ROCm/triton.git)
 
     Install ROCm's Triton following the instructions from [ROCm/triton](https://github.com/ROCm/triton.git)
@@ -252,8 +270,7 @@ You can find more information about vLLM's wheels in
     pip uninstall -y triton
     git clone https://github.com/ROCm/triton.git
     cd triton
-    # git checkout $TRITON_BRANCH
+    git checkout $TRITON_BRANCH
-    git checkout f9e5bf54
     if [ ! -f setup.py ]; then cd python; fi
     python3 setup.py install
     cd ../..
@@ -265,17 +282,14 @@ You can find more information about vLLM's wheels in
 
 2. Optionally, if you choose to use CK flash attention, you can install [flash attention for ROCm](https://github.com/Dao-AILab/flash-attention.git)
 
-    Install ROCm's flash attention (v2.8.0) following the instructions from [ROCm/flash-attention](https://github.com/Dao-AILab/flash-attention#amd-rocm-support)
+    Install ROCm's flash attention following the instructions from [ROCm/flash-attention](https://github.com/Dao-AILab/flash-attention#amd-rocm-support)
-
-    For example, for ROCm 7.0, suppose your gfx arch is `gfx942`. To get your gfx architecture, run `rocminfo |grep gfx`.
 
     ```bash
     git clone https://github.com/Dao-AILab/flash-attention.git
     cd flash-attention
-    # git checkout $FA_BRANCH
+    git checkout $FA_BRANCH
-    git checkout 0e60e394
     git submodule update --init
-    GPU_ARCHS="gfx942" python3 setup.py install
+    GPU_ARCHS="${GPU_ARCH}" python3 setup.py install
     cd ..
     ```
 
@@ -290,7 +304,8 @@ You can find more information about vLLM's wheels in
     cd aiter
     git checkout $AITER_BRANCH_OR_COMMIT
     git submodule sync; git submodule update --init --recursive
-    python3 setup.py develop
+    # Keep the Triton installed in step 1
+    AITER_USE_SYSTEM_TRITON=1 python3 setup.py develop
     ```
 
     !!! note
@@ -300,32 +315,34 @@ You can find more information about vLLM's wheels in
 4. Optionally, if you want to use MORI for EP or PD disaggregation, you can install [MORI](https://github.com/ROCm/mori) using the following steps:
 
     ```bash
+    pip install "setuptools_scm>=6.2"
     git clone https://github.com/ROCm/mori.git
     cd mori
     git checkout $MORI_BRANCH_OR_COMMIT
     git submodule sync; git submodule update --init --recursive
-    MORI_GPU_ARCHS="gfx942;gfx950" python3 setup.py install
+    # The ROCm SDK keeps its CMake configs for system dependencies (e.g. NUMA) under rocm_sysdeps
+    CMAKE_PREFIX_PATH="${ROCM_PATH}:${ROCM_PATH}/lib/rocm_sysdeps" \
+        MORI_GPU_ARCHS="gfx942;gfx950" python3 setup.py install
     ```
 
     !!! note
         - You will need to config the `$MORI_BRANCH_OR_COMMIT` for your purpose.
         - The validated `$MORI_BRANCH_OR_COMMIT` can be found in the [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base).
 
-5. Build vLLM. For example, vLLM on ROCM 7.0 can be built with the following steps:
+5. Build vLLM. For example, vLLM on ROCm 10.0 can be built with the following steps:
 
     ???+ console "Commands"
 
         ```bash
         pip install --upgrade pip
 
-        # Build & install AMD SMI
+        # Install AMD SMI, which the ROCm SDK ships as sources
-        pip install /opt/rocm/share/amd_smi
+        SDK_CORE=$(python3 -c "import _rocm_sdk_core, os; print(os.path.dirname(_rocm_sdk_core.__file__))")
+        pip install ${SDK_CORE}/share/amd_smi
+        # It loads the SDK's libamd_smi relative to its own directory
+        export PYTHONPATH=${SDK_CORE}/share/amd_smi
 
         # Install dependencies
-        pip install --upgrade numba \
-            scipy \
-            huggingface-hub \
-            setuptools_scm
         pip install -r requirements/rocm.txt
 
         # To build for a single architecture (e.g., MI300) for faster installation (recommended):
@@ -352,8 +369,9 @@ You can find more information about vLLM's wheels in
 vLLM offers official Docker images for deployment.
 The images can be used to run OpenAI compatible server and are available on Docker Hub as [vllm/vllm-openai-rocm](https://hub.docker.com/r/vllm/vllm-openai-rocm/tags).
 
-- `vllm/vllm-openai-rocm:latest` — stable release
+- `vllm/vllm-openai-rocm:latest` — stable release, built on ROCm 10.0 (also tagged `latest-rocm100`)
-- `vllm/vllm-openai-rocm:nightly` — preview build from the latest development branch, use this if you want the latest features and fixes
+- `vllm/vllm-openai-rocm:nightly` — preview build from the latest development branch on ROCm 10.0 (also tagged `nightly-rocm100`), use this if you want the latest features and fixes
+- `vllm/vllm-openai-rocm:latest-rocm72` / `nightly-rocm72` — the same builds on the legacy ROCm 7.2 stack, kept for a transition period
 
 ```bash
 docker run --rm \
@@ -409,7 +427,7 @@ You can build and run vLLM from source via the provided [docker/Dockerfile.rocm]
 ??? info "(Optional) Build an image with ROCm software stack"
 
     Build a docker image from [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base) which setup ROCm software stack needed by the vLLM.
-    **This step is optional as this rocm_base image is usually prebuilt and store at [Docker Hub](https://hub.docker.com/r/rocm/vllm-dev) under tag `rocm/vllm-dev:base` to speed up user experience.**
+    **This step is optional as this rocm_base image is usually prebuilt and store at [Docker Hub](https://hub.docker.com/r/rocm/vllm-dev) under tag `rocm/vllm-dev:base` to speed up user experience.** For the legacy ROCm 7.2 stack, the base image is built from [docker/Dockerfile.rocm_72_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_72_base) and published as `vllm/vllm-openai-rocm:base-nightly-rocm72`.
     If you choose to build this rocm_base image yourself, the steps are as follows.
 
     It is important that the user kicks off the docker build using buildkit. Either the user put `DOCKER_BUILDKIT=1` as environment variable when calling docker build command, or the user needs to set up buildkit in the docker daemon configuration `/etc/docker/daemon.json` as follows and restart the daemon:
@@ -422,7 +440,7 @@ You can build and run vLLM from source via the provided [docker/Dockerfile.rocm]
     }
     ```
 
-    To build vllm on ROCm 7.0 for MI200 and MI300 series, you can use the default:
+    To build vllm on ROCm 10.0 for all supported GPUs, you can use the default:
 
     ```bash
     DOCKER_BUILDKIT=1 docker build \
@@ -441,15 +459,15 @@ It is important that the user kicks off the docker build using buildkit. Either
 }
 ```
 
-[docker/Dockerfile.rocm](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm) uses ROCm 7.0 by default, but also supports ROCm 5.7, 6.0, 6.1, 6.2, 6.3, and 6.4, in older vLLM branches.
+[docker/Dockerfile.rocm](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm) uses ROCm 10.0 (installed from TheRock wheels) by default. The legacy ROCm 7.2 stack is available as [docker/Dockerfile.rocm_72](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_72) with [docker/Dockerfile.rocm_72_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_72_base); older vLLM branches support ROCm 5.7 through 7.0.
 It provides flexibility to customize the build of docker image using the following arguments:
 
-- `BASE_IMAGE`: specifies the base image used when running `docker build`. The default value `rocm/vllm-dev:base` is an image published and maintained by AMD. It is being built using [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base)
+- `BASE_IMAGE`: specifies the base image used when running `docker build`. The default value `rocm/vllm-dev:base` is an image published and maintained by AMD. It is being built using [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base). For [docker/Dockerfile.rocm_72](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_72), the default is `vllm/vllm-openai-rocm:base-nightly-rocm72`
 - `ARG_PYTORCH_ROCM_ARCH`: Allows to override the gfx architecture values from the base docker image
 
 Their values can be passed in when running `docker build` with `--build-arg` options.
 
-To build vllm on ROCm 7.0 for MI200 and MI300 series, you can use the default (which build a docker image with `vllm serve` as entrypoint):
+To build vllm on ROCm 10.0 for all supported GPUs, you can use the default (which build a docker image with `vllm serve` as entrypoint):
 
 ```bash
 DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.rocm -t vllm/vllm-openai-rocm .

setup.py

@@ -645,8 +645,10 @@ class precompiled_wheel_utils:
 
     @staticmethod
     def rocm_version_to_variant(rocm_version: str) -> str:
-        """Convert a ROCm version string to a wheel variant, e.g. 7.2.3 -> rocm723."""
+        """Convert a ROCm version string to a wheel variant from its major and
-        return "rocm" + rocm_version.replace(".", "")
+        minor version, e.g. 7.2.3 -> rocm72, 10.0.0 -> rocm100.
+        """
+        return "rocm" + "".join(rocm_version.split(".")[:2])
 
     @staticmethod
     def detect_system_rocm_variant() -> str | None:
@@ -1382,7 +1384,9 @@ def get_vllm_version() -> str:
             # Get the Rocm Version
             rocm_version = get_rocm_version() or torch.version.hip
             if rocm_version and rocm_version != envs.VLLM_MAIN_CUDA_VERSION:
-                version += f"{sep}rocm{rocm_version.replace('.', '')[:3]}"
+                version += sep + precompiled_wheel_utils.rocm_version_to_variant(
+                    rocm_version
+                )
         elif _is_tpu():
             version += f"{sep}tpu"
         elif _is_cpu():
@@ -1512,6 +1516,7 @@ def get_requirements() -> list[str]:
         requirements = _read_requirements("rocm.txt")
         if "develop" in sys.argv[1:] and "--no-deps" not in sys.argv[1:]:
             _check_requirements_preinstalled(requirements)
+        requirements += get_rocm_device_requirements()
     elif _is_tpu():
         requirements = _read_requirements("tpu.txt")
     elif _is_cpu():
@@ -1523,6 +1528,24 @@ def get_requirements() -> list[str]:
     return requirements
 
 
+def get_rocm_device_requirements() -> list[str]:
+    """GPU kernel packages for TheRock builds.
+
+    TheRock ships GPU kernels as separate per-arch wheels selected through
+    ``[device-<arch>]`` extras on torch, torchvision and rocm. Require them for
+    every arch this wheel was compiled for, so ``pip install vllm`` works on
+    any supported GPU.
+    """
+    if not _is_hip() or importlib.util.find_spec("rocm_sdk") is None:
+        return []
+    arches = [a for a in os.getenv("PYTORCH_ROCM_ARCH", "").split(";") if a]
+    return [
+        f"{pkg}[device-{arch}]"
+        for pkg in ("torch", "torchvision", "rocm")
+        for arch in arches
+    ]
+
+
 ext_modules = []
 
 if _is_cuda() or _is_hip():

tests/models/language/pooling/test_token_classification.py

@@ -110,7 +110,7 @@ def test_modernbert_models(
     for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
         hf_output = hf_output.detach().clone().cpu().float()
         vllm_output = vllm_output.detach().clone().cpu().float()
-        torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
+        torch.testing.assert_close(hf_output, vllm_output, atol=6e-2, rtol=1e-3)
 
 
 @pytest.mark.parametrize("model", ["Davlan/xlm-roberta-base-ner-hrl"])

tools/vllm-rocm/generate-rocm-wheels-root-index.sh

@@ -17,14 +17,20 @@
 #
 # Environment variables:
 #   S3_BUCKET   - Bucket name (default: vllm-wheels)
-#   VARIANT     - ROCm variant (default: rocm721)
+#   ROCM_BASE_DOCKERFILE - default ROCm stack whose variant the root index serves
+#                          (default: docker/Dockerfile.rocm_base)
+#   VARIANT     - override the ROCm variant derived from ROCM_BASE_DOCKERFILE
 #   DRY_RUN     - Set to 1 for preview mode (same as --dry-run)
 
 set -euo pipefail
 
 # ======== Configuration ========
 BUCKET="${S3_BUCKET:-vllm-wheels}"
-VARIANT="${VARIANT:-rocm721}"
+if [[ -z "${VARIANT:-}" ]]; then
+    # shellcheck source=.buildkite/scripts/rocm/stack.sh
+    source .buildkite/scripts/rocm/stack.sh "${ROCM_BASE_DOCKERFILE:-docker/Dockerfile.rocm_base}"
+    VARIANT="$ROCM_STACK_VARIANT"
+fi
 DRY_RUN="${DRY_RUN:-0}"
 FORCE_VERSION=""
 

tools/vllm-rocm/therock_wheels.py

@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Adapt TheRock (ROCm 10) wheels for the vLLM ROCm wheel index.
+
+repack-torch: TheRock's torch wheel hard-pins the triton it was built with.
+    vLLM ships a newer source-built triton, so rewrite that one Requires-Dist
+    and the METADATA hash in RECORD. The version and filename stay the same,
+    because torchvision and the amd-torch-device-* wheels pin torch exactly.
+
+external-links: list the TheRock wheels the index links to instead of hosting
+    (ROCm SDK, device kernels, torchvision/torchaudio, rocm-bootstrap), with
+    versions and GPU arches read from docker/Dockerfile.rocm_base. torch's
+    device-<arch> extras can also pull in a GPU-family wheel (e.g.
+    amd-torch-device-gfx110x), so those are read from the torch wheel itself.
+
+patch-amdsmi: rocm-sdk-core ships the amdsmi bindings under
+    _rocm_sdk_core/share/amd_smi, and the loader finds libamd_smi relative to
+    that directory. Once installed as its own package it would miss the SDK
+    library, so also look it up through the installed _rocm_sdk_core package.
+"""
+
+import argparse
+import hashlib
+import shutil
+import sys
+import tempfile
+import urllib.request
+import zipfile
+from html.parser import HTMLParser
+from pathlib import Path
+from urllib.parse import unquote, urljoin
+
+import pybase64 as base64
+import regex as re
+
+
+def _record_hash(data: bytes) -> str:
+    digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=")
+    return f"sha256={digest.decode()}"
+
+
+def repack_torch(wheel: Path, triton_version: str, out_dir: Path) -> Path:
+    with zipfile.ZipFile(wheel) as src:
+        infos = src.infolist()
+        metadata = next(i for i in infos if i.filename.endswith(".dist-info/METADATA"))
+        record = next(i for i in infos if i.filename.endswith(".dist-info/RECORD"))
+
+        text = src.read(metadata).decode()
+        new_text, count = re.subn(
+            r"^Requires-Dist: triton\s*==\s*\S+\s*$",
+            f"Requires-Dist: triton=={triton_version}",
+            text,
+            flags=re.MULTILINE,
+        )
+        if count != 1:
+            raise SystemExit(
+                f"expected exactly one pinned triton requirement, found {count}"
+            )
+        new_metadata = new_text.encode()
+
+        lines = src.read(record).decode().splitlines()
+        prefix = metadata.filename + ","
+        replaced = [
+            f"{metadata.filename},{_record_hash(new_metadata)},{len(new_metadata)}"
+            if line.startswith(prefix)
+            else line
+            for line in lines
+        ]
+        if replaced == lines:
+            raise SystemExit(f"{metadata.filename} not found in RECORD")
+        new_record = ("\n".join(replaced) + "\n").encode()
+
+        out_dir.mkdir(parents=True, exist_ok=True)
+        out = out_dir / wheel.name
+        with zipfile.ZipFile(out, "w") as dst:
+            for info in infos:
+                if info.filename == metadata.filename:
+                    data = new_metadata
+                elif info.filename == record.filename:
+                    data = new_record
+                else:
+                    data = src.read(info)
+                dst.writestr(info, data, compress_type=info.compress_type)
+    return out
+
+
+_AMDSMI_ANCHOR = "    possible_locations.append(libamd_smi_path)\n"
+_AMDSMI_PATCH = """\
+    # vLLM: installed as its own package, find the lib in TheRock's rocm-sdk-core.
+    try:
+        import importlib.util
+        _sdk_core = importlib.util.find_spec("_rocm_sdk_core")
+        if _sdk_core is not None and _sdk_core.origin:
+            possible_locations.append(
+                Path(_sdk_core.origin).parent / "lib/libamd_smi.so.27")
+    except Exception:
+        pass
+"""
+
+
+def patch_amdsmi(src_dir: Path, out_dir: Path) -> Path:
+    out_dir = out_dir.resolve()
+    if out_dir.exists():
+        shutil.rmtree(out_dir)
+    shutil.copytree(src_dir, out_dir, ignore=shutil.ignore_patterns("tests"))
+    wrapper = out_dir / "amdsmi" / "amdsmi_wrapper.py"
+    text = wrapper.read_text()
+    if text.count(_AMDSMI_ANCHOR) != 1:
+        raise SystemExit(f"amdsmi loader anchor not found in {wrapper}")
+    wrapper.write_text(text.replace(_AMDSMI_ANCHOR, _AMDSMI_ANCHOR + _AMDSMI_PATCH))
+    return out_dir
+
+
+def dockerfile_args(dockerfile: Path) -> dict[str, str]:
+    args = {}
+    for m in re.finditer(r'^ARG (\w+)="?([^"\s]*)"?', dockerfile.read_text(), re.M):
+        args.setdefault(m.group(1), m.group(2))
+    return args
+
+
+class _Links(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__()
+        self.hrefs: list[str] = []
+
+    def handle_starttag(self, tag, attrs):
+        if tag == "a":
+            self.hrefs += [v for k, v in attrs if k == "href" and v]
+
+
+def _project_files(index_url: str, project: str) -> list[str]:
+    page = urljoin(index_url, f"{project}/")
+    with urllib.request.urlopen(page, timeout=60) as resp:
+        parser = _Links()
+        parser.feed(resp.read().decode())
+    return [urljoin(page, href.split("#")[0]) for href in parser.hrefs]
+
+
+def _usable(url: str, version: str | None) -> bool:
+    name = unquote(url.rsplit("/", 1)[-1])
+    if (
+        version is not None
+        and f"-{version}-" not in name
+        and f"-{version}." not in name
+    ):
+        return False
+    if name.endswith(".tar.gz"):
+        return True
+    tags = name[: -len(".whl")].split("-")[-3:]
+    return (
+        name.endswith(".whl")
+        and tags[0] in ("cp312", "py3")
+        and (tags[2] == "any" or ("x86_64" in tags[2] and "win" not in tags[2]))
+    )
+
+
+def torch_device_requirements(
+    torch_wheel: Path, arches: list[str]
+) -> list[tuple[str, str]]:
+    """(project, version) pairs that torch's device-<arch> extras require."""
+    with zipfile.ZipFile(torch_wheel) as zf:
+        metadata = next(n for n in zf.namelist() if n.endswith(".dist-info/METADATA"))
+        text = zf.read(metadata).decode()
+    wanted = {f"device-{arch}" for arch in arches}
+    reqs = []
+    for m in re.finditer(
+        r'^Requires-Dist: ([\w.-]+)\s*==\s*(\S+)\s*;.*extra == "([^"]+)"', text, re.M
+    ):
+        if m.group(3) in wanted and (m.group(1), m.group(2)) not in reqs:
+            reqs.append((m.group(1), m.group(2)))
+    missing = wanted - set(re.findall(r'extra == "(device-[^"]+)"', text))
+    if missing:
+        raise SystemExit(f"torch has no extras for {sorted(missing)}")
+    return reqs
+
+
+def external_links(dockerfile: Path, torch_wheel: Path) -> list[str]:
+    args = dockerfile_args(dockerfile)
+    index = args["ROCM_RELEASE_WHEELS_MULTIARCH_URL"]
+    sdk = args["ROCM_SDK_VERSION"]
+    vision_v, audio_v = args["TORCHVISION_VERSION"], args["TORCHAUDIO_VERSION"]
+    arches = [a for a in args["PYTORCH_ROCM_ARCH"].split(";") if a]
+    wanted: list[tuple[str, str | None]] = [
+        ("rocm", sdk),
+        ("rocm-sdk-core", sdk),
+        ("rocm-sdk-libraries", sdk),
+        ("torchvision", vision_v),
+        ("torchaudio", audio_v),
+        ("rocm-bootstrap", None),
+    ]
+    for arch in arches:
+        wanted += [
+            (f"rocm-sdk-device-{arch}", sdk),
+            (f"amd-torchvision-device-{arch}", vision_v),
+        ]
+    wanted += torch_device_requirements(torch_wheel, arches)
+    urls = []
+    for project, version in wanted:
+        found = [u for u in _project_files(index, project) if _usable(u, version)]
+        if not found:
+            raise SystemExit(f"no usable {project}=={version} files on {index}")
+        urls += found
+    return urls
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    sub = parser.add_subparsers(dest="cmd", required=True)
+    p = sub.add_parser("repack-torch")
+    p.add_argument("wheel", type=Path)
+    p.add_argument("--triton-version", required=True)
+    p.add_argument("--out-dir", type=Path, required=True)
+    p = sub.add_parser("external-links")
+    p.add_argument(
+        "--dockerfile", type=Path, default=Path("docker/Dockerfile.rocm_base")
+    )
+    p.add_argument("--torch-wheel", type=Path, required=True)
+    p = sub.add_parser("patch-amdsmi")
+    p.add_argument("src_dir", type=Path)
+    p.add_argument("--out-dir", type=Path, default=None)
+    args = parser.parse_args()
+
+    if args.cmd == "repack-torch":
+        print(repack_torch(args.wheel, args.triton_version, args.out_dir))
+    elif args.cmd == "external-links":
+        print("\n".join(external_links(args.dockerfile, args.torch_wheel)))
+    else:
+        out = args.out_dir or Path(tempfile.mkdtemp()) / "amd_smi"
+        print(patch_amdsmi(args.src_dir, out))
+
+
+if __name__ == "__main__":
+    sys.exit(main())

vllm/platforms/rocm.py

@@ -405,6 +405,12 @@ if (
 ):
     os.environ["HIP_ONLINE_TUNING"] = "1"
 
+# RCCL 2.30's RAS thread can abort the process at exit ("terminate called
+# without an active exception") when its shutdown notification fails.
+# https://github.com/ROCm/rocm-systems/issues/13572
+if torch.version.hip:
+    os.environ.setdefault("NCCL_RAS_ENABLE", "0")
+
 
 @cache
 def use_rocm_custom_paged_attention(