From b2456c622b48ae85af5e55406fb138d9a23364be Mon Sep 17 00:00:00 2001 From: Rohan Potdar Date: Sat, 17 Oct 2026 21:43:59 -0500 Subject: [PATCH] [ROCm][CI] Make ROCm 10.0 the default image; keep ROCm 7.2 as -rocm72 (#58761) Signed-off-by: Rohan Potdar Signed-off-by: Rohan138 Signed-off-by: Micah Williamson Co-authored-by: Claude Opus 5.5 Co-authored-by: Micah Williamson Co-authored-by: Kevin H. Luu Co-authored-by: Andreas Karatzas --- .buildkite/release-pipeline.yaml | 681 ++++++------------ .buildkite/scripts/annotate-rocm-release.sh | 106 +-- .buildkite/scripts/cache-rocm-base-wheels.sh | 62 +- .buildkite/scripts/ci-bake-rocm.sh | 61 +- .buildkite/scripts/cleanup-nightly-builds.sh | 7 +- .buildkite/scripts/generate-nightly-index.py | 72 +- .buildkite/scripts/publish-release-images.sh | 44 +- .../scripts/push-nightly-builds-rocm.sh | 107 ++- .buildkite/scripts/rocm/build-config.sh | 4 +- .buildkite/scripts/rocm/stack.sh | 78 ++ .buildkite/scripts/upload-rocm-wheels.sh | 86 ++- docker/Dockerfile.rock_base | 453 ------------ docker/Dockerfile.rocm | 82 +-- .../{Dockerfile.rock => Dockerfile.rocm_72} | 98 +-- docker/Dockerfile.rocm_72_base | 479 ++++++++++++ docker/Dockerfile.rocm_base | 537 +++++++------- docker/ci-rocm.hcl | 34 +- docker/docker-bake-rocm.hcl | 4 - .../installation/gpu.rocm.inc.md | 100 +-- setup.py | 29 +- .../pooling/test_token_classification.py | 2 +- .../generate-rocm-wheels-root-index.sh | 10 +- tools/vllm-rocm/therock_wheels.py | 234 ++++++ vllm/platforms/rocm.py | 6 + 24 files changed, 1742 insertions(+), 1634 deletions(-) create mode 100644 .buildkite/scripts/rocm/stack.sh delete mode 100644 docker/Dockerfile.rock_base rename docker/{Dockerfile.rock => Dockerfile.rocm_72} (92%) create mode 100644 docker/Dockerfile.rocm_72_base create mode 100644 tools/vllm-rocm/therock_wheels.py diff --git a/.buildkite/release-pipeline.yaml b/.buildkite/release-pipeline.yaml index 6064f93104..84987524b2 100644 --- a/.buildkite/release-pipeline.yaml +++ b/.buildkite/release-pipeline.yaml @@ -703,385 +703,6 @@ steps: DOCKER_BUILDKIT: "1" DOCKERHUB_USERNAME: "vllmbot" - # ============================================================================= - # ROCm Release Pipeline (x86_64 only) - # ============================================================================= - # - # vLLM version is determined by the Buildkite checkout (like CUDA pipeline). - # To build a specific version, trigger the build from that branch/tag. - # - # Environment variables for ROCm builds (set via Buildkite UI or schedule): - # - # Note: ROCm version is determined by BASE_IMAGE in docker/Dockerfile.rocm_base - # - # ============================================================================= - - - group: "Build ROCm Wheel / Image " - key: "build-rocm-wheel-image" - depends_on: ~ - steps: - # ROCm Job 1: Build ROCm Base Wheels (with S3 caching) - - label: ":rocm: Build ROCm Base Image & Wheels" - id: build-rocm-base-wheels - depends_on: ~ - agents: - queue: cpu_queue_release - commands: - - | - set -euo pipefail - - # Generate cache key - CACHE_KEY=$$(.buildkite/scripts/cache-rocm-base-wheels.sh key) - ECR_CACHE_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:$${CACHE_KEY}-rocm-base" - - echo "========================================" - echo "ROCm Base Build Configuration" - echo "========================================" - echo " CACHE_KEY: $${CACHE_KEY}" - echo " ECR_CACHE_TAG: $${ECR_CACHE_TAG}" - echo "========================================" - - # Login to ECR - aws ecr-public get-login-password --region us-east-1 | \ - docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7 - - IMAGE_EXISTS=false - WHEELS_EXIST=false - - # Check ECR for Docker image - - if docker manifest inspect "$${ECR_CACHE_TAG}" > /dev/null 2>&1; then - IMAGE_EXISTS=true - echo "ECR image cache HIT" - fi - - # Check S3 for wheels - WHEEL_CACHE_STATUS=$(.buildkite/scripts/cache-rocm-base-wheels.sh check) - if [ "$${WHEEL_CACHE_STATUS}" = "hit" ]; then - WHEELS_EXIST=true - echo "S3 wheels cache HIT" - fi - - - # Scenario 1: Both cached (best case) - if [ "$${IMAGE_EXISTS}" = "true" ] && [ "$${WHEELS_EXIST}" = "true" ]; then - echo "" - echo "FULL CACHE HIT - Reusing both image and wheels" - echo "" - - # Download wheels - .buildkite/scripts/cache-rocm-base-wheels.sh download - - # Save ECR tag for downstream jobs - buildkite-agent meta-data set "rocm-base-image-tag" "$${ECR_CACHE_TAG}" - - # Scenario 2: Full rebuild needed - else - echo "" - echo " CACHE MISS - Building from scratch..." - echo "" - - # Build full base image and push to ECR - DOCKER_BUILDKIT=1 docker buildx build \ - --file docker/Dockerfile.rocm_base \ - --tag "$${ECR_CACHE_TAG}" \ - --build-arg USE_SCCACHE=1 \ - --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \ - --build-arg SCCACHE_REGION_NAME=us-west-2 \ - --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \ - --push \ - . - - # Build wheel extraction stage - DOCKER_BUILDKIT=1 docker buildx build \ - --file docker/Dockerfile.rocm_base \ - --tag rocm-base-debs:$${BUILDKITE_BUILD_NUMBER} \ - --target debs_wheel_release \ - --build-arg USE_SCCACHE=1 \ - --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \ - --build-arg SCCACHE_REGION_NAME=us-west-2 \ - --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \ - --load \ - . - - # Extract and upload wheels - mkdir -p artifacts/rocm-base-wheels - cid=$(docker create rocm-base-debs:$${BUILDKITE_BUILD_NUMBER}) - docker cp $${cid}:/app/debs/. artifacts/rocm-base-wheels/ - docker rm $${cid} - - .buildkite/scripts/cache-rocm-base-wheels.sh upload - - # Cache base docker image to ECR - docker push "$${ECR_CACHE_TAG}" - - buildkite-agent meta-data set "rocm-base-image-tag" "$${ECR_CACHE_TAG}" - - echo "" - echo " Build complete - Image and wheels cached" - fi - - artifact_paths: - - "artifacts/rocm-base-wheels/*.whl" - env: - DOCKER_BUILDKIT: "1" - S3_BUCKET: "vllm-wheels" - - # ROCm Job 2: Build vLLM ROCm Wheel - - label: ":python: Build vLLM ROCm Wheel - x86_64" - id: build-rocm-vllm-wheel - depends_on: - - step: build-rocm-base-wheels - allow_failure: false - agents: - queue: cpu_queue_release - timeout_in_minutes: 180 - commands: - # Download artifacts and prepare Docker image - - | - set -euo pipefail - - # Ensure git tags are up-to-date (Buildkite's default fetch doesn't update tags) - # This fixes version detection when tags are moved/force-pushed - echo "Fetching latest tags from origin..." - git fetch --tags --force origin - - # Log tag information for debugging version detection - echo "========================================" - echo "Git Tag Verification" - echo "========================================" - echo "Current HEAD: $(git rev-parse HEAD)" - echo "git describe --tags: $(git describe --tags 2>/dev/null || echo 'No tags found')" - echo "" - echo "Recent tags (pointing to commits near HEAD):" - git tag -l --sort=-creatordate | head -5 - echo "setuptools_scm version detection:" - pip install -q setuptools_scm 2>/dev/null || true - python3 -c "import setuptools_scm; print(' Detected version:', setuptools_scm.get_version())" 2>/dev/null || echo " (setuptools_scm not available in this environment)" - echo "========================================" - - # Download wheel artifacts from current build - echo "Downloading wheel artifacts from current build" - buildkite-agent artifact download "artifacts/rocm-base-wheels/*.whl" . - - # Get ECR image tag from metadata (set by build-rocm-base-wheels) - ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rocm-base-image-tag 2>/dev/null || echo '')" - if [ -z "$${ECR_IMAGE_TAG}" ]; then - echo "ERROR: rocm-base-image-tag metadata not found" - echo "This should have been set by the build-rocm-base-wheels job" - exit 1 - fi - - echo "Pulling base Docker image from ECR: $${ECR_IMAGE_TAG}" - - # Login to ECR - aws ecr-public get-login-password --region us-east-1 | \ - docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7 - - # Pull base Docker image from ECR - docker pull "$${ECR_IMAGE_TAG}" - - echo "Loaded base image: $${ECR_IMAGE_TAG}" - - # Prepare base wheels for Docker build context - mkdir -p docker/context/base-wheels - touch docker/context/base-wheels/.keep - cp artifacts/rocm-base-wheels/*.whl docker/context/base-wheels/ - echo "Base wheels for vLLM build:" - ls -lh docker/context/base-wheels/ - - echo "========================================" - echo "Building vLLM wheel with:" - echo " BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}" - echo " BUILDKITE_BRANCH: $${BUILDKITE_BRANCH}" - echo " BASE_IMAGE: $${ECR_IMAGE_TAG}" - echo "========================================" - - # Build vLLM wheel using local checkout (REMOTE_VLLM=0) - DOCKER_BUILDKIT=1 docker build \ - --file docker/Dockerfile.rocm \ - --target export_vllm_wheel_release \ - --output type=local,dest=rocm-dist \ - --build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \ - --build-arg REMOTE_VLLM=0 \ - --build-arg GIT_REPO_CHECK=1 \ - --build-arg USE_SCCACHE=1 \ - --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \ - --build-arg SCCACHE_REGION_NAME=us-west-2 \ - --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \ - . - echo "Built vLLM wheel:" - ls -lh rocm-dist/*.whl - # Copy wheel to artifacts directory - mkdir -p artifacts/rocm-vllm-wheel - cp rocm-dist/*.whl artifacts/rocm-vllm-wheel/ - echo "Final vLLM wheel:" - ls -lh artifacts/rocm-vllm-wheel/ - artifact_paths: - - "artifacts/rocm-vllm-wheel/*.whl" - env: - DOCKER_BUILDKIT: "1" - S3_BUCKET: "vllm-wheels" - - # ROCm Job 3: Upload Wheels to S3 - - label: ":s3: Upload ROCm Wheels to S3" - id: upload-rocm-wheels - depends_on: - - step: build-rocm-vllm-wheel - allow_failure: false - agents: - queue: cpu_queue_release - timeout_in_minutes: 60 - commands: - # Download all wheel artifacts and run upload - - | - set -euo pipefail - - # Download artifacts from current build - echo "Downloading artifacts from current build" - buildkite-agent artifact download "artifacts/rocm-base-wheels/*.whl" . - buildkite-agent artifact download "artifacts/rocm-vllm-wheel/*.whl" . - - # # Run upload script - bash .buildkite/scripts/upload-rocm-wheels.sh - env: - DOCKER_BUILDKIT: "1" - S3_BUCKET: "vllm-wheels" - - # ROCm Job 4: Annotate ROCm Wheel Release - - label: ":memo: Annotate ROCm wheel release" - id: annotate-rocm-release - depends_on: - - upload-rocm-wheels - agents: - queue: cpu_queue_release - commands: - - "bash .buildkite/scripts/annotate-rocm-release.sh" - env: - S3_BUCKET: "vllm-wheels" - - # ROCm Job 5: Generate Root Index for ROCm Wheels (for release only) - # This is the job to create https://wheels.vllm.ai/rocm/ index allowing - # users to install with `uv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm/` - - block: "Generate Root Index for ROCm Wheels for Release" - key: block-generate-root-index-rocm-wheels - depends_on: upload-rocm-wheels - - - label: ":package: Generate Root Index for ROCm Wheels for Release" - depends_on: block-generate-root-index-rocm-wheels - id: generate-root-index-rocm-wheels - agents: - queue: cpu_queue_release - commands: - - "bash tools/vllm-rocm/generate-rocm-wheels-root-index.sh" - env: - S3_BUCKET: "vllm-wheels" - VARIANT: "rocm723" - - # ROCm Job 6: Build ROCm Release Docker Image - - label: ":docker: Build release image - x86_64 - ROCm" - id: build-rocm-release-image - depends_on: - - step: block-build-release-images - allow_failure: true - - step: build-rocm-base-wheels - allow_failure: false - agents: - queue: cpu_queue_release - timeout_in_minutes: 60 - commands: - - | - set -euo pipefail - - # Login to ECR - aws ecr-public get-login-password --region us-east-1 | \ - docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7 - - # Get ECR image tag from metadata (set by build-rocm-base-wheels) - ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rocm-base-image-tag 2>/dev/null || echo '')" - if [ -z "$${ECR_IMAGE_TAG}" ]; then - echo "ERROR: rocm-base-image-tag metadata not found" - echo "This should have been set by the build-rocm-base-wheels job" - exit 1 - fi - - echo "Pulling base Docker image from ECR: $${ECR_IMAGE_TAG}" - - # Pull base Docker image from ECR - docker pull "$${ECR_IMAGE_TAG}" - - echo "Loaded base image: $${ECR_IMAGE_TAG}" - - # Pass the base image ECR tag to downstream steps (nightly publish) - buildkite-agent meta-data set "rocm-base-ecr-tag" "$${ECR_IMAGE_TAG}" - - echo "========================================" - echo "Building vLLM ROCm release image with:" - echo " BASE_IMAGE: $${ECR_IMAGE_TAG}" - echo " BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}" - echo "========================================" - - # Build vLLM ROCm release image using cached base - DOCKER_BUILDKIT=1 docker build \ - --build-arg max_jobs=16 \ - --build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \ - --build-arg USE_SCCACHE=1 \ - --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \ - --build-arg SCCACHE_REGION_NAME=us-west-2 \ - --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \ - --build-arg INSTALL_LMCACHE=true \ - --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm \ - --target vllm-openai \ - --progress plain \ - -f docker/Dockerfile.rocm . - - # Push to ECR - docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm - - echo "" - echo " Successfully built and pushed ROCm release image" - echo " Image: public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rocm" - echo "" - env: - DOCKER_BUILDKIT: "1" - S3_BUCKET: "vllm-wheels" - - - label: "Publish nightly XPU image to DockerHub" - depends_on: - - create-manifest-xpu - if: build.env("NIGHTLY") == "1" - agents: - queue: small_cpu_queue_release - commands: - - "bash .buildkite/scripts/xpu/push-nightly-builds-xpu.sh" - - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly- vllm/vllm-openai-xpu" - plugins: - - docker-login#v3.0.0: - username: vllmbot - password-env: DOCKERHUB_TOKEN - env: - DOCKER_BUILDKIT: "1" - DOCKERHUB_USERNAME: "vllmbot" - - - label: "Publish nightly ROCm image to DockerHub" - depends_on: - - build-rocm-release-image - if: build.env("NIGHTLY") == "1" - agents: - queue: small_cpu_queue_release - commands: - - "bash .buildkite/scripts/push-nightly-builds-rocm.sh" - # Clean up old nightly builds (keep only last 14) - - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly- vllm/vllm-openai-rocm" - - "bash .buildkite/scripts/cleanup-nightly-builds.sh base-nightly- vllm/vllm-openai-rocm" - plugins: - - docker-login#v3.0.0: - username: vllmbot - password-env: DOCKERHUB_TOKEN - env: - DOCKER_BUILDKIT: "1" - DOCKERHUB_USERNAME: "vllmbot" - - trigger: "perf-eval" label: "Trigger perf-eval (CUDA)" key: trigger-perf-eval-cuda @@ -1103,7 +724,7 @@ steps: key: trigger-perf-eval-rocm if: build.env("NIGHTLY") == "1" || build.env("RUN_PERF_EVAL") == "1" depends_on: - - build-rocm-release-image + - build-rocm100-release-image async: true soft_fail: true build: @@ -1112,138 +733,250 @@ steps: env: NIGHTLY: 1 # run all perf-eval workloads VLLM_COMMIT: "$BUILDKITE_COMMIT" - VLLM_IMAGE_ROCM: "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-rocm" + VLLM_IMAGE_ROCM: "public.ecr.aws/q9t5s3a7/vllm-release-repo:$BUILDKITE_COMMIT-rocm100" # ============================================================================= - # ROCk Release Pipeline (x86_64 only) + # ROCm Release Pipelines (x86_64 only) # ============================================================================= # - # Same shape as the ROCm pipeline above (Jobs 1 and 6 only), but built from - # docker/Dockerfile.rock_base and docker/Dockerfile.rock, which install the - # ROCm SDK from TheRock wheels instead of the ROCm apt repos. + # One group per ROCm stack, identified by its base Dockerfile (ROCM_BASE_DOCKERFILE). + # The scripts derive the ROCm version, wheel variant, cache keys and image tags from + # it (.buildkite/scripts/rocm/stack.sh), so both groups share one set of step + # definitions, anchored on the ROCm 10.0 group. # - # Note: ROCm SDK version is determined by ROCM_SDK_VERSION in - # docker/Dockerfile.rock_base + # - ROCm 10.0 (TheRock, docker/Dockerfile.rocm{,_base}) is the default: it owns the + # :nightly / :latest images and the root wheel index. + # - ROCm 7.2 (docker/Dockerfile.rocm_72{,_base}) is kept for a transition period as + # rocm72 images and wheels. # + # vLLM version is determined by the Buildkite checkout (like the CUDA pipeline). # ============================================================================= - - group: "Build ROCk Image" - key: "build-rock-image" + - group: "Build ROCm 10.0 Wheel / Image" + key: "build-rocm100" depends_on: ~ steps: - # ROCk Job 1: Build ROCk Base Image (with ECR caching) - - label: ":rocm: Build ROCk Base Image" - id: build-rock-base-image + - label: ":rocm: Build ROCm 10.0 Base Image & Wheels" + id: build-rocm100-base depends_on: ~ - agents: + agents: &rocm_build_agents queue: cpu_queue_release timeout_in_minutes: 360 - commands: + env: &rocm100_env + ROCM_BASE_DOCKERFILE: "docker/Dockerfile.rocm_base" + ROCM_DEFAULT_ARG: "--default" + DOCKER_BUILDKIT: "1" + S3_BUCKET: "vllm-wheels" + DOCKERHUB_USERNAME: "vllmbot" + commands: &rocm_base_commands - | set -euo pipefail - - # The SDK version, torch wheels and component branches all live in - # the Dockerfile, so its hash is the cache key. - CACHE_KEY=$$(sha256sum docker/Dockerfile.rock_base | cut -c1-16) - ECR_CACHE_TAG="public.ecr.aws/q9t5s3a7/vllm-release-repo:$${CACHE_KEY}-rock-base" - - echo "========================================" - echo "ROCk Base Build Configuration" - echo "========================================" - echo " CACHE_KEY: $${CACHE_KEY}" - echo " ECR_CACHE_TAG: $${ECR_CACHE_TAG}" - echo "========================================" - - # Login to ECR + source .buildkite/scripts/rocm/stack.sh "$${ROCM_BASE_DOCKERFILE}" + SCCACHE_ARGS=(--build-arg USE_SCCACHE=1 --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache + --build-arg SCCACHE_REGION_NAME=us-west-2 --build-arg SCCACHE_S3_NO_CREDENTIALS=0) aws ecr-public get-login-password --region us-east-1 | \ docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7 - if docker manifest inspect "$${ECR_CACHE_TAG}" > /dev/null 2>&1; then - echo "ECR image cache HIT" + # Base image, cached in ECR by the base Dockerfile's hash + if docker manifest inspect "$${ROCM_STACK_ECR_BASE}" > /dev/null 2>&1; then + echo "ECR image cache HIT: $${ROCM_STACK_ECR_BASE}" else - echo "CACHE MISS - Building from scratch..." - - DOCKER_BUILDKIT=1 docker buildx build \ - --file docker/Dockerfile.rock_base \ - --tag "$${ECR_CACHE_TAG}" \ - --build-arg USE_SCCACHE=1 \ - --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \ - --build-arg SCCACHE_REGION_NAME=us-west-2 \ - --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \ - --push \ - . + docker buildx build --file "$${ROCM_BASE_DOCKERFILE}" --tag "$${ROCM_STACK_ECR_BASE}" \ + "$${SCCACHE_ARGS[@]}" --push . fi - # Save ECR tag for downstream jobs - buildkite-agent meta-data set "rock-base-image-tag" "$${ECR_CACHE_TAG}" - env: - DOCKER_BUILDKIT: "1" + # Base wheels, cached in S3 (see cache-rocm-base-wheels.sh) + CACHE=.buildkite/scripts/cache-rocm-base-wheels.sh + if [ "$$($${CACHE} "$${ROCM_BASE_DOCKERFILE}" check)" = "hit" ]; then + $${CACHE} "$${ROCM_BASE_DOCKERFILE}" download + else + DEBS="rocm-base-debs:$${BUILDKITE_BUILD_NUMBER}-$${ROCM_STACK_VARIANT}" + docker buildx build --file "$${ROCM_BASE_DOCKERFILE}" --tag "$${DEBS}" \ + --target debs_wheel_release "$${SCCACHE_ARGS[@]}" --load . + mkdir -p "$${ROCM_STACK_BASE_WHEELS_DIR}" + cid=$$(docker create "$${DEBS}") + docker cp "$${cid}":/app/debs/. "$${ROCM_STACK_BASE_WHEELS_DIR}/" + docker rm "$${cid}" + $${CACHE} "$${ROCM_BASE_DOCKERFILE}" upload + fi + buildkite-agent artifact upload "$${ROCM_STACK_BASE_WHEELS_DIR}/*.whl" - # ROCk Job 2: Build ROCk Release Docker Image - - label: ":docker: Build release image - x86_64 - ROCk" - id: build-rock-release-image + - label: ":python: Build vLLM ROCm 10.0 Wheel - x86_64" + id: build-rocm100-vllm-wheel + depends_on: build-rocm100-base + agents: *rocm_build_agents + timeout_in_minutes: 180 + env: *rocm100_env + commands: &rocm_vllm_wheel_commands + - | + set -euo pipefail + source .buildkite/scripts/rocm/stack.sh "$${ROCM_BASE_DOCKERFILE}" + # Buildkite's default fetch doesn't update moved tags; version detection needs them + git fetch --tags --force origin + buildkite-agent artifact download "$${ROCM_STACK_BASE_WHEELS_DIR}/*.whl" . + aws ecr-public get-login-password --region us-east-1 | \ + docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7 + docker pull "$${ROCM_STACK_ECR_BASE}" + mkdir -p docker/context/base-wheels + touch docker/context/base-wheels/.keep + cp "$${ROCM_STACK_BASE_WHEELS_DIR}"/*.whl docker/context/base-wheels/ + docker build --file "$${ROCM_STACK_DOCKERFILE}" --target export_vllm_wheel_release \ + --output type=local,dest=rocm-dist \ + --build-arg BASE_IMAGE="$${ROCM_STACK_ECR_BASE}" --build-arg REMOTE_VLLM=0 \ + --build-arg GIT_REPO_CHECK=1 --build-arg USE_SCCACHE=1 \ + --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \ + --build-arg SCCACHE_REGION_NAME=us-west-2 --build-arg SCCACHE_S3_NO_CREDENTIALS=0 . + mkdir -p "$${ROCM_STACK_VLLM_WHEEL_DIR}" + cp rocm-dist/*.whl "$${ROCM_STACK_VLLM_WHEEL_DIR}/" + buildkite-agent artifact upload "$${ROCM_STACK_VLLM_WHEEL_DIR}/*.whl" + + - label: ":s3: Upload ROCm 10.0 Wheels to S3" + id: upload-rocm100-wheels + depends_on: build-rocm100-vllm-wheel + agents: *rocm_build_agents + timeout_in_minutes: 60 + env: *rocm100_env + commands: &rocm_upload_commands + - | + set -euo pipefail + source .buildkite/scripts/rocm/stack.sh "$${ROCM_BASE_DOCKERFILE}" + buildkite-agent artifact download "$${ROCM_STACK_BASE_WHEELS_DIR}/*.whl" . + buildkite-agent artifact download "$${ROCM_STACK_VLLM_WHEEL_DIR}/*.whl" . + bash .buildkite/scripts/upload-rocm-wheels.sh "$${ROCM_BASE_DOCKERFILE}" + + - label: ":memo: Annotate ROCm 10.0 wheel release" + id: annotate-rocm100-release + depends_on: upload-rocm100-wheels + agents: *rocm_build_agents + env: *rocm100_env + commands: &rocm_annotate_commands + - bash .buildkite/scripts/annotate-rocm-release.sh "$${ROCM_BASE_DOCKERFILE}" $${ROCM_DEFAULT_ARG} + + # Generate https://wheels.vllm.ai/rocm/ (the default stack's variant), so users can + # `uv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm/` (release only) + - block: "Generate Root Index for ROCm Wheels for Release" + key: block-generate-root-index-rocm-wheels + depends_on: + - upload-rocm100-wheels + - upload-rocm72-wheels + + - label: ":package: Generate Root Index for ROCm Wheels for Release" + depends_on: block-generate-root-index-rocm-wheels + id: generate-root-index-rocm-wheels + agents: *rocm_build_agents + env: *rocm100_env + commands: + - "bash tools/vllm-rocm/generate-rocm-wheels-root-index.sh" + + - label: ":docker: Build release image - x86_64 - ROCm 10.0" + id: build-rocm100-release-image depends_on: - step: block-build-release-images allow_failure: true - - step: build-rock-base-image + - step: build-rocm100-base allow_failure: false - agents: - queue: cpu_queue_release + agents: *rocm_build_agents timeout_in_minutes: 180 - commands: + env: *rocm100_env + commands: &rocm_release_image_commands - | set -euo pipefail - - # Login to ECR + source .buildkite/scripts/rocm/stack.sh "$${ROCM_BASE_DOCKERFILE}" aws ecr-public get-login-password --region us-east-1 | \ docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7 - - ECR_IMAGE_TAG="$$(buildkite-agent meta-data get rock-base-image-tag 2>/dev/null || echo '')" - if [ -z "$${ECR_IMAGE_TAG}" ]; then - echo "ERROR: rock-base-image-tag metadata not found" - echo "This should have been set by the build-rock-base-image job" - exit 1 - fi - - docker pull "$${ECR_IMAGE_TAG}" - - # Pass the base image ECR tag to downstream steps (nightly publish) - buildkite-agent meta-data set "rock-base-ecr-tag" "$${ECR_IMAGE_TAG}" - - echo "========================================" - echo "Building vLLM ROCk release image with:" - echo " BASE_IMAGE: $${ECR_IMAGE_TAG}" - echo " BUILDKITE_COMMIT: $${BUILDKITE_COMMIT}" - echo "========================================" - - DOCKER_BUILDKIT=1 docker build \ - --build-arg max_jobs=16 \ - --build-arg BASE_IMAGE="$${ECR_IMAGE_TAG}" \ - --build-arg USE_SCCACHE=1 \ + docker pull "$${ROCM_STACK_ECR_BASE}" + docker build --file "$${ROCM_STACK_DOCKERFILE}" --target vllm-openai \ + --tag "$${ROCM_STACK_ECR_IMAGE}" --progress plain \ + --build-arg max_jobs=16 --build-arg BASE_IMAGE="$${ROCM_STACK_ECR_BASE}" \ + --build-arg INSTALL_LMCACHE=true --build-arg USE_SCCACHE=1 \ --build-arg SCCACHE_BUCKET_NAME=vllm-build-sccache \ - --build-arg SCCACHE_REGION_NAME=us-west-2 \ - --build-arg SCCACHE_S3_NO_CREDENTIALS=0 \ - --build-arg INSTALL_LMCACHE=true \ - --tag public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rock \ - --target vllm-openai \ - --progress plain \ - -f docker/Dockerfile.rock . + --build-arg SCCACHE_REGION_NAME=us-west-2 --build-arg SCCACHE_S3_NO_CREDENTIALS=0 . + docker push "$${ROCM_STACK_ECR_IMAGE}" - docker push public.ecr.aws/q9t5s3a7/vllm-release-repo:$${BUILDKITE_COMMIT}-rock - env: + - label: "Publish nightly ROCm 10.0 image to DockerHub" + id: publish-rocm100-nightly + depends_on: build-rocm100-release-image + if: build.env("NIGHTLY") == "1" + agents: &rocm_publish_agents + queue: small_cpu_queue_release + env: *rocm100_env + commands: &rocm_publish_nightly_commands + - bash .buildkite/scripts/push-nightly-builds-rocm.sh "$${ROCM_BASE_DOCKERFILE}" $${ROCM_DEFAULT_ARG} + plugins: &rocm_dockerhub_login + - docker-login#v3.0.0: + username: vllmbot + password-env: DOCKERHUB_TOKEN + + - group: "Build ROCm 7.2 Wheel / Image" + key: "build-rocm72" + depends_on: ~ + steps: + - label: ":rocm: Build ROCm 7.2 Base Image & Wheels" + id: build-rocm72-base + depends_on: ~ + agents: *rocm_build_agents + timeout_in_minutes: 360 + env: &rocm72_env + ROCM_BASE_DOCKERFILE: "docker/Dockerfile.rocm_72_base" + ROCM_DEFAULT_ARG: "" DOCKER_BUILDKIT: "1" + S3_BUCKET: "vllm-wheels" + DOCKERHUB_USERNAME: "vllmbot" + commands: *rocm_base_commands - - label: "Publish nightly ROCk image to DockerHub" + - label: ":python: Build vLLM ROCm 7.2 Wheel - x86_64" + id: build-rocm72-vllm-wheel + depends_on: build-rocm72-base + agents: *rocm_build_agents + timeout_in_minutes: 180 + env: *rocm72_env + commands: *rocm_vllm_wheel_commands + + - label: ":s3: Upload ROCm 7.2 Wheels to S3" + id: upload-rocm72-wheels + depends_on: build-rocm72-vllm-wheel + agents: *rocm_build_agents + timeout_in_minutes: 60 + env: *rocm72_env + commands: *rocm_upload_commands + + - label: ":memo: Annotate ROCm 7.2 wheel release" + id: annotate-rocm72-release + depends_on: upload-rocm72-wheels + agents: *rocm_build_agents + env: *rocm72_env + commands: *rocm_annotate_commands + + - label: ":docker: Build release image - x86_64 - ROCm 7.2" + id: build-rocm72-release-image depends_on: - - build-rock-release-image + - step: block-build-release-images + allow_failure: true + - step: build-rocm72-base + allow_failure: false + agents: *rocm_build_agents + timeout_in_minutes: 180 + env: *rocm72_env + commands: *rocm_release_image_commands + + - label: "Publish nightly ROCm 7.2 image to DockerHub" + id: publish-rocm72-nightly + depends_on: build-rocm72-release-image + if: build.env("NIGHTLY") == "1" + agents: *rocm_publish_agents + env: *rocm72_env + commands: *rocm_publish_nightly_commands + plugins: *rocm_dockerhub_login + + - label: "Publish nightly XPU image to DockerHub" + depends_on: + - create-manifest-xpu if: build.env("NIGHTLY") == "1" agents: queue: small_cpu_queue_release commands: - - "bash .buildkite/scripts/push-nightly-builds-rocm.sh rock rocm100" - # Clean up old nightly builds (keep only last 14) - - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly-rocm100- vllm/vllm-openai-rocm" - - "bash .buildkite/scripts/cleanup-nightly-builds.sh base-nightly-rocm100- vllm/vllm-openai-rocm" + - "bash .buildkite/scripts/xpu/push-nightly-builds-xpu.sh" + - "bash .buildkite/scripts/cleanup-nightly-builds.sh nightly- vllm/vllm-openai-xpu" plugins: - docker-login#v3.0.0: username: vllmbot @@ -1264,7 +997,8 @@ steps: - create-multi-arch-manifest-ubuntu2404 - create-multi-arch-manifest-cuda-12-9-ubuntu2404 - create-manifest-xpu - - build-rocm-release-image + - build-rocm100-release-image + - build-rocm72-release-image - input-release-version # Wait for CPU builds if their block steps were unblocked, so publish # doesn't race the in-progress CPU build. allow_failure lets publish @@ -1289,6 +1023,7 @@ steps: - "cuda-13-0-ubuntu-24-04" - "cuda-12-9-ubuntu-24-04" - "rocm" + - "rocm72" - "xpu" - "cpu" plugins: diff --git a/.buildkite/scripts/annotate-rocm-release.sh b/.buildkite/scripts/annotate-rocm-release.sh index d661297227..1889db43b5 100755 --- a/.buildkite/scripts/annotate-rocm-release.sh +++ b/.buildkite/scripts/annotate-rocm-release.sh @@ -2,15 +2,23 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # -# Generate Buildkite annotation for ROCm wheel release +# Generate Buildkite annotation for a ROCm stack's wheel release. +# +# Usage: annotate-rocm-release.sh [--default] +# base-dockerfile e.g. docker/Dockerfile.rocm_base or docker/Dockerfile.rocm_72_base +# --default the stack also owns the plain :latest / :v Docker Hub tags set -ex -# Extract build configuration from Dockerfile.rocm_base (single source of truth) -# Extract ROCm version dynamically from Dockerfile.rocm_base -# BASE_IMAGE format: rocm/dev-ubuntu-22.04:7.0-complete -> extracts "7.0" -ROCM_VERSION=$(grep -E '^ARG BASE_IMAGE=' docker/Dockerfile.rocm_base | sed -E 's/.*:([0-9]+\.[0-9]+).*/\1/' || echo "unknown") -PYTHON_VERSION=$(grep '^ARG PYTHON_VERSION=' docker/Dockerfile.rocm_base | sed 's/^ARG PYTHON_VERSION=//') -PYTORCH_ROCM_ARCH=$(grep '^ARG PYTORCH_ROCM_ARCH=' docker/Dockerfile.rocm_base | sed 's/^ARG PYTORCH_ROCM_ARCH=//') +# shellcheck source=.buildkite/scripts/rocm/stack.sh +source "$(dirname "${BASH_SOURCE[0]}")/rocm/stack.sh" "${1:?Usage: $0 [--default]}" +TAG_VARIANTS=("-${ROCM_STACK_VARIANT}") +if [[ "${2:-}" == "--default" ]]; then + TAG_VARIANTS=("" "-${ROCM_STACK_VARIANT}") +fi +ROCM_VERSION="$ROCM_STACK_VERSION" +VARIANT="$ROCM_STACK_VARIANT" +PYTHON_VERSION=$(sed -nE 's/^ARG PYTHON_VERSION="?([^" ]+)"?.*/\1/p' "$ROCM_STACK_BASE_DOCKERFILE") +PYTORCH_ROCM_ARCH=$(sed -nE 's/^ARG PYTORCH_ROCM_ARCH="?([^"]+)"?.*/\1/p' "$ROCM_STACK_BASE_DOCKERFILE") # Get release version, default to 1.0.0.dev for nightly/per-commit builds RELEASE_VERSION=$(buildkite-agent meta-data get release-version 2>/dev/null || echo "") @@ -18,22 +26,32 @@ if [ -z "${RELEASE_VERSION}" ]; then RELEASE_VERSION="1.0.0.dev" fi -ROCM_BASE_CACHE_KEY=$(.buildkite/scripts/cache-rocm-base-wheels.sh key) - # S3 URLs S3_BUCKET="${S3_BUCKET:-vllm-wheels}" S3_REGION="${AWS_DEFAULT_REGION:-us-west-2}" -S3_URL="http://${S3_BUCKET}.s3-website-${S3_REGION}.amazonaws.com" +S3_HOST="${S3_BUCKET}.s3-website-${S3_REGION}.amazonaws.com" +S3_URL="http://${S3_HOST}" +WHEEL_PATH="rocm/${BUILDKITE_COMMIT}/${VARIANT}-wheels/" -# Format ROCm version for path (e.g., "7.1" -> "rocm710") -ROCM_VERSION_PATH="rocm$(echo "${ROCM_VERSION}" | tr -d '.')" -ROCM_PATH="rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}" -buildkite-agent annotate --style 'success' --context 'rocm-release-workflow' << EOF -## ROCm Wheel and Docker Image Releases +DOCKER_CMDS="docker pull ${ROCM_STACK_ECR_BASE} +docker pull ${ROCM_STACK_ECR_IMAGE}" +for suffix in "${TAG_VARIANTS[@]}"; do + for tag in "latest${suffix}" "v${RELEASE_VERSION}${suffix}"; do + DOCKER_CMDS+=" +docker tag ${ROCM_STACK_ECR_BASE} vllm/vllm-openai-rocm:${tag}-base +docker tag ${ROCM_STACK_ECR_IMAGE} vllm/vllm-openai-rocm:${tag} +docker push vllm/vllm-openai-rocm:${tag}-base +docker push vllm/vllm-openai-rocm:${tag}" + done +done + +buildkite-agent annotate --style 'success' --context "rocm-release-workflow-${VARIANT}" << EOF +## ROCm ${ROCM_VERSION} Wheel and Docker Image Releases ### Build Configuration | Setting | Value | |---------|-------| | **ROCm Version** | ${ROCM_VERSION} | +| **Index Variant** | ${VARIANT} | | **Python Version** | ${PYTHON_VERSION} | | **GPU Architectures** | ${PYTORCH_ROCM_ARCH} | | **Branch** | \`${BUILDKITE_BRANCH}\` | @@ -41,49 +59,20 @@ buildkite-agent annotate --style 'success' --context 'rocm-release-workflow' << ### :package: Installation -**Install from this build (by commit):** - \`\`\`bash -pip install vllm --extra-index-url ${S3_URL}/${ROCM_PATH}/ --trusted-host ${S3_BUCKET}.s3-website-${S3_REGION}.amazonaws.com - -# Example for ROCm ${ROCM_VERSION}: -pip install vllm --extra-index-url ${S3_URL}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/ --trusted-host ${S3_BUCKET}.s3-website-${S3_REGION}.amazonaws.com -\`\`\` - -**Install from nightly (if published):** - -\`\`\`bash -pip install vllm --extra-index-url ${S3_URL}/rocm/nightly/ --trusted-host ${S3_BUCKET}.s3-website-${S3_REGION}.amazonaws.com +# This build (by commit) +pip install vllm --extra-index-url ${S3_URL}/rocm/${BUILDKITE_COMMIT}/${VARIANT}/ --trusted-host ${S3_HOST} +# Nightly (if published) +pip install vllm --extra-index-url ${S3_URL}/rocm/nightly/${VARIANT}/ --trusted-host ${S3_HOST} \`\`\` ### :floppy_disk: Download Wheels Directly \`\`\`bash -# List all ROCm wheels -aws s3 ls s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/ -# Download specific wheels -aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/vllm-*.whl . -aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/torch-*.whl . -aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/triton-*.whl . -aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/triton-kernels-*.whl . -aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/torchvision-*.whl . -aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/torchaudio-*.whl . -aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/amdsmi-*.whl . -aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/amd_aiter-*.whl . -aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/flash-attn-*.whl . +aws s3 ls s3://${S3_BUCKET}/${WHEEL_PATH} +aws s3 cp --recursive --exclude "*" --include "*.whl" s3://${S3_BUCKET}/${WHEEL_PATH} . \`\`\` -### :gear: Included Packages -- **vllm**: vLLM with ROCm support -- **torch**: PyTorch built for ROCm ${ROCM_VERSION} -- **triton**: Triton -- **triton-kernels**: Triton kernels -- **torchvision**: TorchVision for ROCm PyTorch -- **torchaudio**: Torchaudio for ROCm PyTorch -- **amdsmi**: AMD SMI Python bindings -- **amd_aiter**: Aiter for ROCm -- **flash-attn**: Flash Attention for ROCm - ### :warning: Notes - These wheels are built for **ROCm ${ROCM_VERSION}** and will NOT work with CUDA GPUs - Supported GPU architectures: ${PYTORCH_ROCM_ARCH} @@ -91,23 +80,8 @@ aws s3 cp s3://${S3_BUCKET}/rocm/${BUILDKITE_COMMIT}/${ROCM_VERSION_PATH}/flash- ### :package: Docker Image Release -To download and upload the image: - \`\`\` -docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm-base -docker pull public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm - -docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base -docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base vllm/vllm-openai-rocm:latest-base -docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT}-base vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base -docker push vllm/vllm-openai-rocm:latest-base -docker push vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base - -docker tag public.ecr.aws/q9t5s3a7/vllm-release-repo:${BUILDKITE_COMMIT}-rocm vllm/vllm-openai-rocm:${BUILDKITE_COMMIT} -docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT} vllm/vllm-openai-rocm:latest -docker tag vllm/vllm-openai-rocm:${BUILDKITE_COMMIT} vllm/vllm-openai-rocm:v${RELEASE_VERSION} -docker push vllm/vllm-openai-rocm:latest -docker push vllm/vllm-openai-rocm:v${RELEASE_VERSION} +${DOCKER_CMDS} \`\`\` EOF diff --git a/.buildkite/scripts/cache-rocm-base-wheels.sh b/.buildkite/scripts/cache-rocm-base-wheels.sh index dc76544d36..02fbbf52cf 100755 --- a/.buildkite/scripts/cache-rocm-base-wheels.sh +++ b/.buildkite/scripts/cache-rocm-base-wheels.sh @@ -5,39 +5,33 @@ # Cache helper for ROCm base wheels # # This script manages caching of pre-built ROCm base wheels (torch, triton, etc.) -# to avoid rebuilding them when Dockerfile.rocm_base hasn't changed. +# to avoid rebuilding them when the stack's base Dockerfile hasn't changed. # -# Usage: -# cache-rocm-base-wheels.sh check - Check if cache exists, outputs "hit" or "miss" -# cache-rocm-base-wheels.sh upload - Upload wheels to cache -# cache-rocm-base-wheels.sh download - Download wheels from cache -# cache-rocm-base-wheels.sh key - Output the cache key +# Usage: cache-rocm-base-wheels.sh +# base-dockerfile e.g. docker/Dockerfile.rocm_base or docker/Dockerfile.rocm_72_base +# Commands: +# check - Check if cache exists, outputs "hit" or "miss" +# upload - Upload wheels to cache +# download - Download wheels from cache +# key - Output the cache key +# path - Output the full S3 cache path # # Environment variables: # S3_BUCKET - S3 bucket name (default: vllm-wheels) # -# Note: ROCm version is determined by BASE_IMAGE in Dockerfile.rocm_base, -# so changes to ROCm version are captured by the Dockerfile hash. +# The cache key hashes the base Dockerfile and the repo files it COPYs in, which +# capture the ROCm version and every component pin (see rocm/stack.sh). set -euo pipefail BUCKET="${S3_BUCKET:-vllm-wheels}" -DOCKERFILE="docker/Dockerfile.rocm_base" CACHE_PREFIX="rocm/cache" -# Generate hash from Dockerfile content + build args -generate_cache_key() { - # Include Dockerfile content - if [[ ! -f "$DOCKERFILE" ]]; then - echo "ERROR: Dockerfile not found: $DOCKERFILE" >&2 - exit 1 - fi - local dockerfile_hash=$(sha256sum "$DOCKERFILE" | cut -c1-16) - - echo "${dockerfile_hash}" -} - -CACHE_KEY=$(generate_cache_key) +# shellcheck source=.buildkite/scripts/rocm/stack.sh +source "$(dirname "${BASH_SOURCE[0]}")/rocm/stack.sh" "${1:?Usage: $0 }" +shift +CACHE_KEY="$ROCM_STACK_WHEEL_KEY" +WHEELS_DIR="$ROCM_STACK_BASE_WHEELS_DIR" CACHE_PATH="s3://${BUCKET}/${CACHE_PREFIX}/${CACHE_KEY}/" case "${1:-}" in @@ -67,19 +61,19 @@ case "${1:-}" in echo "Cache path: ${CACHE_PATH}" echo "" - if [[ ! -d "artifacts/rocm-base-wheels" ]]; then - echo "ERROR: artifacts/rocm-base-wheels directory not found" >&2 + if [[ ! -d "${WHEELS_DIR}" ]]; then + echo "ERROR: ${WHEELS_DIR} directory not found" >&2 exit 1 fi - WHEEL_COUNT=$(find artifacts/rocm-base-wheels -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l) + WHEEL_COUNT=$(find "${WHEELS_DIR}" -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l) if [[ "$WHEEL_COUNT" -eq 0 ]]; then - echo "ERROR: No wheels found in artifacts/rocm-base-wheels/" >&2 + echo "ERROR: No wheels found in ${WHEELS_DIR}/" >&2 exit 1 fi echo "Uploading $WHEEL_COUNT wheels..." - aws s3 cp --recursive artifacts/rocm-base-wheels/ "${CACHE_PATH}" + aws s3 cp --recursive "${WHEELS_DIR}/" "${CACHE_PATH}" echo "" echo "Cache upload complete!" @@ -93,17 +87,17 @@ case "${1:-}" in echo "Cache key: ${CACHE_KEY}" echo "Cache path: ${CACHE_PATH}" echo "" - mkdir -p artifacts/rocm-base-wheels + mkdir -p "${WHEELS_DIR}" # Use sync with include/exclude to only download .whl files - aws s3 sync "${CACHE_PATH}" artifacts/rocm-base-wheels/ \ + aws s3 sync "${CACHE_PATH}" "${WHEELS_DIR}/" \ --exclude "*" \ --include "*.whl" echo "" echo "Downloaded wheels:" - find artifacts/rocm-base-wheels -maxdepth 1 -name '*.whl' -exec ls -lh {} \; - WHEEL_COUNT=$(find artifacts/rocm-base-wheels -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l) + find "${WHEELS_DIR}" -maxdepth 1 -name '*.whl' -exec ls -lh {} \; + WHEEL_COUNT=$(find "${WHEELS_DIR}" -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l) echo "" echo "Total: $WHEEL_COUNT wheels" echo "========================================" @@ -118,12 +112,12 @@ case "${1:-}" in ;; *) - echo "Usage: $0 {check|upload|download|key|path}" >&2 + echo "Usage: $0 {check|upload|download|key|path}" >&2 echo "" >&2 echo "Commands:" >&2 echo " check - Check if cache exists, outputs 'hit' or 'miss'" >&2 - echo " upload - Upload wheels from artifacts/rocm-base-wheels/ to cache" >&2 - echo " download - Download wheels from cache to artifacts/rocm-base-wheels/" >&2 + echo " upload - Upload wheels from ${WHEELS_DIR}/ to cache" >&2 + echo " download - Download wheels from cache to ${WHEELS_DIR}/" >&2 echo " key - Output the cache key" >&2 echo " path - Output the full S3 cache path" >&2 exit 1 diff --git a/.buildkite/scripts/ci-bake-rocm.sh b/.buildkite/scripts/ci-bake-rocm.sh index ae81f59e84..5935938616 100644 --- a/.buildkite/scripts/ci-bake-rocm.sh +++ b/.buildkite/scripts/ci-bake-rocm.sh @@ -17,7 +17,7 @@ DEFAULT_REPO_SLUG="vllm-project/vllm" DEFAULT_CI_HCL_SOURCE="docker/ci-rocm.hcl" DEFAULT_CI_BASE_CONTENT_FILES=".dockerignore requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt tools/install_torchcodec_rocm.sh rust-toolchain.toml tests/vllm_test_utils" DEFAULT_CI_BASE_DOCKERFILE="docker/Dockerfile.rocm" -DEFAULT_CI_BASE_DOCKERFILE_STAGES="base rust_toolchain_input_0 rust-toolchain-input rust-toolchain build_nixl lmcache_source build_lmcache build_rocshmem build_deepep mori_base ci_base" +DEFAULT_CI_BASE_DOCKERFILE_STAGES="base rust_toolchain_input_0 rust-toolchain-input rust-toolchain build_nixl lmcache_source build_lmcache build_deepep mori_base ci_base" DEFAULT_CI_BASE_METADATA_VERSION="3" # ROCm CI forces REMOTE_VLLM=0, so content identity covers only the selected # local-source stages rather than unreachable remote-fetch alternatives. @@ -555,9 +555,6 @@ configure_custom_rocm_stages() { CI_BASE_DOCKERFILE_STAGES="${CI_BASE_DOCKERFILE_STAGES:-${stages}}" ROCM_CSRC_DOCKERFILE_STAGES="${ROCM_CSRC_DOCKERFILE_STAGES:-${stages}}" ROCM_RUST_DOCKERFILE_STAGES="${ROCM_RUST_DOCKERFILE_STAGES:-${stages}}" - if [[ " ${stages} " != *" build_rocshmem "* ]]; then - unset ROCSHMEM_BRANCH ROCSHMEM_CACHE_KEY DEEPEP_CACHE_KEY - fi } compose_dependency_cache_key() { @@ -1724,14 +1721,11 @@ ci_base_metadata_pairs() { metadata_pair "vllm.rocm.nixl_commit" "${NIXL_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "NIXL_BRANCH")}" metadata_pair "vllm.rocm.ucx_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "UCX_REPO")" metadata_pair "vllm.rocm.ucx_commit" "${UCX_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "UCX_BRANCH")}" - metadata_pair "vllm.rocm.rocshmem_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "ROCSHMEM_REPO")" - metadata_pair "vllm.rocm.rocshmem_commit" "${ROCSHMEM_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "ROCSHMEM_BRANCH")}" metadata_pair "vllm.rocm.deepep_repo" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_REPO")" metadata_pair "vllm.rocm.deepep_commit" "${DEEPEP_BRANCH:-$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_BRANCH")}" metadata_pair "vllm.rocm.deepep_nic" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_NIC")" metadata_pair "vllm.rocm.deepep_rocm_arch" "$(resolve_dockerfile_arg_value "${dockerfile}" "DEEPEP_ROCM_ARCH")" metadata_pair "vllm.rocm.nixl_cache_key" "${NIXL_CACHE_KEY:-}" - metadata_pair "vllm.rocm.rocshmem_cache_key" "${ROCSHMEM_CACHE_KEY:-}" metadata_pair "vllm.rocm.deepep_cache_key" "${DEEPEP_CACHE_KEY:-}" } @@ -2314,7 +2308,7 @@ extract_dependency_pins() { return 0 fi - for var in NIXL_BRANCH UCX_BRANCH ROCSHMEM_BRANCH DEEPEP_BRANCH; do + for var in NIXL_BRANCH UCX_BRANCH DEEPEP_BRANCH; do if [[ -n "${!var:-}" ]]; then echo "Using provided ${var}: ${!var}" continue @@ -2336,17 +2330,14 @@ compute_dependency_cache_keys() { local dockerfile_rocm="" local nixl_branch="" local ucx_branch="" - local rocshmem_branch="" local deepep_branch="" local nixl_material="" - local rocshmem_material="" local deepep_material="" bake_dir=$(dirname "${VLLM_BAKE_FILE}") dockerfile_rocm="${CI_BASE_DOCKERFILE:-${bake_dir}/Dockerfile.rocm}" nixl_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "NIXL_BRANCH") ucx_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "UCX_BRANCH") - rocshmem_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "ROCSHMEM_BRANCH") deepep_branch=$(resolve_dockerfile_arg_value "${dockerfile_rocm}" "DEEPEP_BRANCH") if [[ -n "${nixl_branch}" && -n "${ucx_branch}" ]]; then @@ -2360,22 +2351,11 @@ compute_dependency_cache_keys() { echo "NIXL dependency cache key: ${NIXL_CACHE_KEY}" fi - if [[ -n "${rocshmem_branch}" ]]; then - rocshmem_material=$(compose_stage_cache_material "${dockerfile_rocm}" "base build_rocshmem") - ROCSHMEM_CACHE_KEY=$( - compose_dependency_cache_key \ - "${rocshmem_branch}" \ - "${rocshmem_material}" - ) - export ROCSHMEM_CACHE_KEY - echo "ROCShmem dependency cache key: ${ROCSHMEM_CACHE_KEY}" - fi - - if [[ -n "${deepep_branch}" && -n "${rocshmem_branch}" ]]; then - deepep_material=$(compose_stage_cache_material "${dockerfile_rocm}" "base build_rocshmem build_deepep") + if [[ -n "${deepep_branch}" ]]; then + deepep_material=$(compose_stage_cache_material "${dockerfile_rocm}" "base build_deepep") DEEPEP_CACHE_KEY=$( compose_dependency_cache_key \ - "${deepep_branch}-rocshmem-${rocshmem_branch}" \ + "${deepep_branch}" \ "${deepep_material}" ) export DEEPEP_CACHE_KEY @@ -2415,18 +2395,11 @@ dependency_cache_ref_for_target() { printf '%s\n' "${cache_repo}:nixl-rocm-${NIXL_BRANCH}-ucx-${UCX_BRANCH:-}" fi ;; - rocshmem-rocm-ci) - if [[ -n "${ROCSHMEM_CACHE_KEY:-}" ]]; then - printf '%s\n' "${cache_repo}:rocshmem-rocm-${ROCSHMEM_CACHE_KEY}" - elif [[ -n "${ROCSHMEM_BRANCH:-}" ]]; then - printf '%s\n' "${cache_repo}:rocshmem-rocm-${ROCSHMEM_BRANCH}" - fi - ;; deepep-rocm-ci) if [[ -n "${DEEPEP_CACHE_KEY:-}" ]]; then printf '%s\n' "${cache_repo}:deepep-rocm-${DEEPEP_CACHE_KEY}" elif [[ -n "${DEEPEP_BRANCH:-}" ]]; then - printf '%s\n' "${cache_repo}:deepep-rocm-${DEEPEP_BRANCH}-rocshmem-${ROCSHMEM_BRANCH:-}" + printf '%s\n' "${cache_repo}:deepep-rocm-${DEEPEP_BRANCH}" fi ;; esac @@ -2444,7 +2417,6 @@ add_dependency_cache_target() { resolve_ci_base_dependency_targets() { local mode="${ROCM_DEP_CACHE_EXPORT_MODE:-missing}" local nixl_ref="" - local rocshmem_ref="" local deepep_ref="" [[ "${TARGET}" == "ci-base-rocm-ci-with-deps" ]] || return 0 @@ -2452,7 +2424,7 @@ resolve_ci_base_dependency_targets() { case "${mode}" in always) echo "ROCM_DEP_CACHE_EXPORT_MODE=always; exporting all dependency caches serially" - for target in nixl-rocm-ci rocshmem-rocm-ci deepep-rocm-ci; do + for target in nixl-rocm-ci deepep-rocm-ci; do if [[ -n "$(dependency_cache_ref_for_target "${target}")" ]]; then add_dependency_cache_target "${target}" fi @@ -2482,16 +2454,6 @@ resolve_ci_base_dependency_targets() { fi fi - if [[ "${mode}" != "always" && -n "${ROCSHMEM_CACHE_KEY:-}" ]]; then - rocshmem_ref=$(dependency_cache_ref_for_target "rocshmem-rocm-ci") - if dependency_cache_ref_exists "${rocshmem_ref}"; then - echo "ROCShmem dependency cache exists: ${rocshmem_ref}" - else - echo "ROCShmem dependency cache missing; will seed: ${rocshmem_ref}" - add_dependency_cache_target "rocshmem-rocm-ci" - fi - fi - if [[ "${mode}" != "always" && -n "${DEEPEP_CACHE_KEY:-}" ]]; then deepep_ref=$(dependency_cache_ref_for_target "deepep-rocm-ci") if dependency_cache_ref_exists "${deepep_ref}"; then @@ -2502,15 +2464,6 @@ resolve_ci_base_dependency_targets() { fi fi - # DeepEP inherits from ROCShmem. If ROCShmem is being seeded, seed DeepEP too - # so the pair stays consistent for future ci_base rebuilds. - if printf '%s\n' "${DEPENDENCY_CACHE_TARGETS[@]}" | grep -qx "rocshmem-rocm-ci" \ - && ! printf '%s\n' "${DEPENDENCY_CACHE_TARGETS[@]}" | grep -qx "deepep-rocm-ci" \ - && [[ -n "${DEEPEP_BRANCH:-}" ]]; then - echo "ROCShmem cache is missing; also seeding DeepEP cache" - add_dependency_cache_target "deepep-rocm-ci" - fi - BAKE_TARGETS=("ci-base-rocm-ci") if [[ ${#DEPENDENCY_CACHE_TARGETS[@]} -eq 0 ]]; then echo "All dependency caches exist; building ci_base without dependency cache exports" diff --git a/.buildkite/scripts/cleanup-nightly-builds.sh b/.buildkite/scripts/cleanup-nightly-builds.sh index 41b04996e4..925ae66dd6 100755 --- a/.buildkite/scripts/cleanup-nightly-builds.sh +++ b/.buildkite/scripts/cleanup-nightly-builds.sh @@ -5,14 +5,17 @@ set -ex # Clean up old nightly builds from DockerHub, keeping only the last 14 builds # This script uses DockerHub API to list and delete old tags with specified prefix # Tags starting with "nightly-dev" are always excluded from deletion -# Usage: cleanup-nightly-builds.sh [TAG_PREFIX] [REPO] +# Usage: cleanup-nightly-builds.sh [TAG_PREFIX] [REPO] [EXCLUDE_PREFIX] # Example: cleanup-nightly-builds.sh "nightly-" # Example: cleanup-nightly-builds.sh "cu130-nightly-" # Example: cleanup-nightly-builds.sh "nightly-" "vllm/vllm-openai-rocm" +# Example: cleanup-nightly-builds.sh "nightly-" "vllm/vllm-openai-rocm" "nightly-rocm" +# (EXCLUDE_PREFIX skips tags owned by another variant sharing TAG_PREFIX) # Get tag prefix and repo from arguments TAG_PREFIX="${1:-nightly-}" REPO="${2:-vllm/vllm-openai}" +EXCLUDE_PREFIX="${3:-}" echo "Cleaning up tags with prefix: $TAG_PREFIX in repository: $REPO" @@ -57,7 +60,7 @@ get_all_tags() { # Get both last_updated timestamp and tag name, separated by | # Exclude mutable architecture tags and nightly-dev tags from cleanup - local tags=$(echo "$response" | jq -r --arg prefix "$TAG_PREFIX" '.results[] | select(.name | startswith($prefix)) | select(.name != ($prefix + "x86_64") and .name != ($prefix + "aarch64")) | select(.name | startswith("nightly-dev") | not) | "\(.last_updated)|\(.name)"') + local tags=$(echo "$response" | jq -r --arg prefix "$TAG_PREFIX" --arg exclude "$EXCLUDE_PREFIX" '.results[] | select(.name | startswith($prefix)) | select($exclude == "" or (.name | startswith($exclude) | not)) | select(.name != ($prefix + "x86_64") and .name != ($prefix + "aarch64")) | select(.name | startswith("nightly-dev") | not) | "\(.last_updated)|\(.name)"') if [ -z "$tags" ]; then break diff --git a/.buildkite/scripts/generate-nightly-index.py b/.buildkite/scripts/generate-nightly-index.py index 8f92996ff9..d7bb1c24e8 100644 --- a/.buildkite/scripts/generate-nightly-index.py +++ b/.buildkite/scripts/generate-nightly-index.py @@ -12,7 +12,7 @@ from dataclasses import asdict, dataclass from datetime import datetime from pathlib import Path from typing import Any -from urllib.parse import quote +from urllib.parse import quote, unquote, urlparse import regex as re @@ -51,6 +51,8 @@ class WheelFileInfo: platform_tag: str variant: str | None filename: str + # Absolute URL for files hosted elsewhere (see --external-links) + url: str | None = None def parse_from_filename(file: str) -> WheelFileInfo: @@ -110,6 +112,29 @@ def parse_from_filename(file: str) -> WheelFileInfo: ) +def parse_external_url(url: str) -> WheelFileInfo: + """Parse an absolute wheel or sdist URL hosted on another index.""" + filename = unquote(urlparse(url).path.rsplit("/", 1)[-1]) + if filename.endswith(".whl"): + info = parse_from_filename(filename) + else: + match = re.match(r"^(?P.+)-(?P[^-]+)\.(tar\.gz|zip)$", filename) + if not match: + raise ValueError(f"Unsupported external file: {filename}") + info = WheelFileInfo( + package_name=match.group("name"), + version=match.group("version"), + build_tag=None, + python_tag="source", + abi_tag="none", + platform_tag="any", + variant=None, + filename=filename, + ) + info.url = url + return info + + def generate_project_list(subdir_names: list[str], comment: str = "") -> str: """Generate project list HTML content linking to each project & variant subdirectory.""" href_tags = [] @@ -129,14 +154,19 @@ def generate_package_index_and_metadata( href_tags = [] metadata = [] for file in sorted(wheel_files, key=lambda x: x.filename): - relative_path = ( - wheel_base_dir.relative_to(index_base_dir, walk_up=True) / file.filename - ) - # handle with '+' in URL, and avoid double-encoding '/' and already-encoded '%2B' - # NOTE: this is AWS S3 specific behavior! - file_path_quoted = quote(relative_path.as_posix(), safe=":%/") + if file.url: + file_path_quoted = file.url + else: + relative_path = ( + wheel_base_dir.relative_to(index_base_dir, walk_up=True) / file.filename + ) + # handle with '+' in URL, and avoid double-encoding '/' and already-encoded '%2B' + # NOTE: this is AWS S3 specific behavior! + file_path_quoted = quote(relative_path.as_posix(), safe=":%/") href_tags.append(f' {file.filename}
') file_meta = asdict(file) + if file_meta["url"] is None: + del file_meta["url"] file_meta["path"] = file_path_quoted metadata.append(file_meta) index_str = INDEX_HTML_TEMPLATE.format(items="\n".join(href_tags), comment=comment) @@ -151,6 +181,7 @@ def generate_index_and_metadata( default_variant: str | None = None, alias_to_default: str | None = None, comment: str = "", + external_urls: list[str] | None = None, ): """Generate index for all wheel files. @@ -160,6 +191,8 @@ def generate_index_and_metadata( index_base_dir (Path): Base directory to store index files. default_variant (str | None): The default variant name, if any. alias_to_default (str | None): Alias variant name for the default variant, if any. + external_urls (list[str] | None): Absolute URLs of files hosted on another index, + listed in the generated indices without being copied. comment (str | None): Optional comment to include in the generated HTML files. First, parse all wheel files to extract metadata. @@ -214,6 +247,7 @@ def generate_index_and_metadata( """ parsed_files = [parse_from_filename(f) for f in whl_files] + parsed_files += [parse_external_url(u) for u in external_urls or []] if not parsed_files: print("No wheel files found, skipping index generation.") @@ -232,10 +266,14 @@ def generate_index_and_metadata( print(f"Detected ROCm variant from vllm: {rocm_variant}") break - # Apply ROCm variant to all wheels without a variant + # Apply ROCm variant to all wheels without a variant. Dependencies with their + # own ROCm local version (e.g. TheRock's torch-2.12.0+rocm10.0.0) belong to + # the same variant rather than one of their own. if rocm_variant: for file in parsed_files: - if file.variant is None: + if file.variant is None or ( + file.variant.startswith("rocm") and file.variant != rocm_variant + ): file.variant = rocm_variant print(f"Inherited variant '{rocm_variant}' for {file.filename}") @@ -366,6 +404,13 @@ if __name__ == "__main__": default=None, help="Alias variant name for the default variant", ) + parser.add_argument( + "--external-links", + type=str, + default=None, + help="Optional file with one absolute URL per line for packages hosted on " + "another index (listed alongside, but not copied into, this index)", + ) parser.add_argument( "--comment", type=str, @@ -463,5 +508,14 @@ if __name__ == "__main__": default_variant=None, alias_to_default=args.alias_to_default, comment=args.comment.strip(), + external_urls=( + [ + line.strip() + for line in Path(args.external_links).read_text().splitlines() + if line.strip() and not line.startswith("#") + ] + if args.external_links + else None + ), ) print(f"Successfully generated index and metadata in {output_dir}") diff --git a/.buildkite/scripts/publish-release-images.sh b/.buildkite/scripts/publish-release-images.sh index dec366390b..81dee9a4df 100755 --- a/.buildkite/scripts/publish-release-images.sh +++ b/.buildkite/scripts/publish-release-images.sh @@ -11,9 +11,9 @@ set -euo pipefail TARGET="${1:-all}" case "${TARGET}" in cuda-13-0 | cuda-12-9 | cuda-13-0-ubuntu-24-04 | \ - cuda-12-9-ubuntu-24-04 | rocm | xpu | cpu | all) ;; + cuda-12-9-ubuntu-24-04 | rocm | rocm72 | xpu | cpu | all) ;; *) - echo "Usage: $0 {cuda-13-0|cuda-12-9|cuda-13-0-ubuntu-24-04|cuda-12-9-ubuntu-24-04|rocm|xpu|cpu|all}" + echo "Usage: $0 {cuda-13-0|cuda-12-9|cuda-13-0-ubuntu-24-04|cuda-12-9-ubuntu-24-04|rocm|rocm72|xpu|cpu|all}" exit 2 ;; esac @@ -143,23 +143,35 @@ if target_enabled cuda-12-9-ubuntu-24-04; then fi # ---- ROCm ---- +# Each ROCm stack is identified by its base Dockerfile (see rocm/stack.sh). The +# default stack also owns the plain :latest / :v tags. + +publish_rocm_stack() { + local base_dockerfile="$1" default="$2" suffix + ( + # shellcheck source=.buildkite/scripts/rocm/stack.sh + source .buildkite/scripts/rocm/stack.sh "$base_dockerfile" + local suffixes=("-${ROCM_STACK_VARIANT}") + [[ "$default" == "1" ]] && suffixes=("" "-${ROCM_STACK_VARIANT}") + docker pull "$ROCM_STACK_ECR_IMAGE" + docker pull "$ROCM_STACK_ECR_BASE" + for suffix in "${suffixes[@]}"; do + for tag in "latest${suffix}" "v${RELEASE_VERSION}${suffix}"; do + docker tag "$ROCM_STACK_ECR_IMAGE" "vllm/vllm-openai-rocm:${tag}" + docker tag "$ROCM_STACK_ECR_BASE" "vllm/vllm-openai-rocm:${tag}-base" + docker push "vllm/vllm-openai-rocm:${tag}" + docker push "vllm/vllm-openai-rocm:${tag}-base" + done + done + ) +} if target_enabled rocm; then - ROCM_BASE_CACHE_KEY=$(.buildkite/scripts/cache-rocm-base-wheels.sh key) - echo "ROCm base cache key: ${ROCM_BASE_CACHE_KEY}" + publish_rocm_stack docker/Dockerfile.rocm_base 1 +fi - docker pull "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm" - docker pull "public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base" - - docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm" vllm/vllm-openai-rocm:latest - docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${COMMIT}-rocm" "vllm/vllm-openai-rocm:v${RELEASE_VERSION}" - docker push vllm/vllm-openai-rocm:latest - docker push "vllm/vllm-openai-rocm:v${RELEASE_VERSION}" - - docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base" vllm/vllm-openai-rocm:latest-base - docker tag "public.ecr.aws/q9t5s3a7/vllm-release-repo:${ROCM_BASE_CACHE_KEY}-rocm-base" "vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base" - docker push vllm/vllm-openai-rocm:latest-base - docker push "vllm/vllm-openai-rocm:v${RELEASE_VERSION}-base" +if target_enabled rocm72; then + publish_rocm_stack docker/Dockerfile.rocm_72_base 0 fi # ---- XPU ---- diff --git a/.buildkite/scripts/push-nightly-builds-rocm.sh b/.buildkite/scripts/push-nightly-builds-rocm.sh index 2a1a0b0804..8b120883ef 100644 --- a/.buildkite/scripts/push-nightly-builds-rocm.sh +++ b/.buildkite/scripts/push-nightly-builds-rocm.sh @@ -2,42 +2,36 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # -# Push a ROCm-family nightly base image and nightly image from ECR to Docker Hub -# under vllm/vllm-openai-rocm, as base-nightly[-], nightly[-] -# and their - forms. -# Run when NIGHTLY=1 after the matching build-*-release-image step has pushed to ECR. +# Push a ROCm stack's nightly base image and nightly image from ECR to Docker Hub +# under vllm/vllm-openai-rocm, as base-nightly-, nightly- and their +# - forms, then prune old nightly tags. With --default, also push the plain +# base-nightly and nightly tags for the same images. +# Run when NIGHTLY=1 after the stack's release image has been pushed to ECR. # -# Usage: push-nightly-builds-rocm.sh [ECR_TAG_SUFFIX TAG_VARIANT] -# Pass both arguments or neither. With none, the default ROCm apt build is -# published as :nightly. The TheRock build passes "rock" (the source ECR tag -# suffix) and "rocm714" (the Docker Hub tag flavor), giving :nightly-rocm714. +# Usage: push-nightly-builds-rocm.sh [--default] +# e.g. "docker/Dockerfile.rocm_base --default" (ROCm 10.0, also :nightly) +# and "docker/Dockerfile.rocm_72_base" (ROCm 7.2, :nightly-rocm72). # # Local testing (no push to Docker Hub): -# BASE_ECR_IMAGE= \ -# BUILDKITE_COMMIT= DRY_RUN=1 \ -# bash .buildkite/scripts/push-nightly-builds-rocm.sh +# BUILDKITE_COMMIT= DRY_RUN=1 \ +# bash .buildkite/scripts/push-nightly-builds-rocm.sh docker/Dockerfile.rocm_base --default # Requires: AWS CLI configured (for ECR public login), Docker. For full run: Docker Hub login. set -euxo pipefail usage() { - echo "Usage: $0 [ECR_TAG_SUFFIX TAG_VARIANT]" >&2 + echo "Usage: $0 [--default]" >&2 } -DOCKERHUB_REPO="vllm/vllm-openai-rocm" - +PUSH_DEFAULT=0 case "$#" in - 0) - ECR_TAG_SUFFIX="rocm" - TAG_VARIANT="" - ;; + 1) ;; 2) - if [[ -z "$1" || -z "$2" ]]; then + if [[ "$2" != "--default" ]]; then usage exit 2 fi - ECR_TAG_SUFFIX="$1" - TAG_VARIANT="$2" + PUSH_DEFAULT=1 ;; *) usage @@ -49,61 +43,44 @@ esac BUILDKITE_COMMIT="${BUILDKITE_COMMIT:?Set BUILDKITE_COMMIT to the commit SHA that has the image in ECR (e.g. from a previous release pipeline run)}" DRY_RUN="${DRY_RUN:-0}" -ECR_REPO="public.ecr.aws/q9t5s3a7/vllm-release-repo" +# shellcheck source=.buildkite/scripts/rocm/stack.sh +source "$(dirname "${BASH_SOURCE[0]}")/rocm/stack.sh" "$1" +DOCKERHUB_REPO="vllm/vllm-openai-rocm" +VARIANT="$ROCM_STACK_VARIANT" -# Get the base image ECR tag (set by the build-*-release-image pipeline step) -BASE_METADATA_KEY="${ECR_TAG_SUFFIX}-base-ecr-tag" -if [[ -n "${BASE_ECR_IMAGE:-}" ]]; then - if [[ "$DRY_RUN" != "1" ]]; then - echo "ERROR: BASE_ECR_IMAGE may only be used with DRY_RUN=1" >&2 - exit 1 - fi - BASE_ORIG_TAG="$BASE_ECR_IMAGE" -elif ! BASE_ORIG_TAG="$(buildkite-agent meta-data get "$BASE_METADATA_KEY")"; then - echo "ERROR: Failed to read required Buildkite metadata '$BASE_METADATA_KEY'" >&2 - exit 1 -fi -if [[ -z "$BASE_ORIG_TAG" ]]; then - echo "ERROR: Required Buildkite metadata '$BASE_METADATA_KEY' is empty" >&2 - echo "Set BASE_ECR_IMAGE to the full ECR base image reference for local testing" >&2 - exit 1 +BASE_TAGS=("base-nightly-${VARIANT}" "base-nightly-${VARIANT}-${BUILDKITE_COMMIT}") +TAGS=("nightly-${VARIANT}" "nightly-${VARIANT}-${BUILDKITE_COMMIT}") +if [[ "$PUSH_DEFAULT" == "1" ]]; then + BASE_TAGS+=("base-nightly" "base-nightly-${BUILDKITE_COMMIT}") + TAGS+=("nightly" "nightly-${BUILDKITE_COMMIT}") fi -ORIG_TAG="${BUILDKITE_COMMIT}-${ECR_TAG_SUFFIX}" -VARIANT_SUFFIX="" -[[ -n "$TAG_VARIANT" ]] && VARIANT_SUFFIX="-$TAG_VARIANT" -BASE_TAG_NAME="base-nightly${VARIANT_SUFFIX}" -TAG_NAME="nightly${VARIANT_SUFFIX}" -BASE_TAG_NAME_COMMIT="base-nightly${VARIANT_SUFFIX}-${BUILDKITE_COMMIT}" -TAG_NAME_COMMIT="nightly${VARIANT_SUFFIX}-${BUILDKITE_COMMIT}" +echo "Pushing base image $ROCM_STACK_ECR_BASE as: ${BASE_TAGS[*]}" +echo "Pushing release image $ROCM_STACK_ECR_IMAGE as: ${TAGS[*]}" -echo "Pushing base image from ECR: $BASE_ORIG_TAG" -echo "Pushing release image from ECR tag: $ORIG_TAG to Docker Hub as $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT" -[[ "$DRY_RUN" == "1" ]] && echo "[DRY_RUN] Skipping push to Docker Hub" - -# Login to ECR and pull the images built by build-*-release-image +# Login to ECR and pull the stack's images aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7 -docker pull "$BASE_ORIG_TAG" -docker pull "${ECR_REPO}:${ORIG_TAG}" +docker pull "$ROCM_STACK_ECR_BASE" +docker pull "$ROCM_STACK_ECR_IMAGE" -# Tag for Docker Hub (base-nightly and base-nightly-, nightly and nightly-) -docker tag "$BASE_ORIG_TAG" "$DOCKERHUB_REPO":"$BASE_TAG_NAME" -docker tag "$BASE_ORIG_TAG" "$DOCKERHUB_REPO":"$BASE_TAG_NAME_COMMIT" -docker tag "${ECR_REPO}:${ORIG_TAG}" "$DOCKERHUB_REPO":"$TAG_NAME" -docker tag "${ECR_REPO}:${ORIG_TAG}" "$DOCKERHUB_REPO":"$TAG_NAME_COMMIT" +for tag in "${BASE_TAGS[@]}"; do docker tag "$ROCM_STACK_ECR_BASE" "$DOCKERHUB_REPO:$tag"; done +for tag in "${TAGS[@]}"; do docker tag "$ROCM_STACK_ECR_IMAGE" "$DOCKERHUB_REPO:$tag"; done if [[ "$DRY_RUN" == "1" ]]; then - echo "[DRY_RUN] Would push $DOCKERHUB_REPO:$BASE_TAG_NAME and $DOCKERHUB_REPO:$BASE_TAG_NAME_COMMIT" - echo "[DRY_RUN] Would push $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT" echo "[DRY_RUN] Local tags created. Exiting without push." exit 0 fi # Push to Docker Hub (docker-login plugin runs before this step in CI) -docker push "$DOCKERHUB_REPO":"$BASE_TAG_NAME" -docker push "$DOCKERHUB_REPO":"$BASE_TAG_NAME_COMMIT" -docker push "$DOCKERHUB_REPO":"$TAG_NAME" -docker push "$DOCKERHUB_REPO":"$TAG_NAME_COMMIT" +for tag in "${BASE_TAGS[@]}" "${TAGS[@]}"; do docker push "$DOCKERHUB_REPO:$tag"; done +echo "Pushed $DOCKERHUB_REPO: ${BASE_TAGS[*]} ${TAGS[*]}" -echo "Pushed $DOCKERHUB_REPO:$BASE_TAG_NAME and $DOCKERHUB_REPO:$BASE_TAG_NAME_COMMIT" -echo "Pushed $DOCKERHUB_REPO:$TAG_NAME and $DOCKERHUB_REPO:$TAG_NAME_COMMIT" +# Keep only the last 14 builds of each tag family. The plain nightly- families +# exclude the variant ones sharing their prefix; commit SHAs never start with "rocm". +CLEANUP=.buildkite/scripts/cleanup-nightly-builds.sh +bash "$CLEANUP" "nightly-${VARIANT}-" "$DOCKERHUB_REPO" +bash "$CLEANUP" "base-nightly-${VARIANT}-" "$DOCKERHUB_REPO" +if [[ "$PUSH_DEFAULT" == "1" ]]; then + bash "$CLEANUP" "nightly-" "$DOCKERHUB_REPO" "nightly-rocm" + bash "$CLEANUP" "base-nightly-" "$DOCKERHUB_REPO" "base-nightly-rocm" +fi diff --git a/.buildkite/scripts/rocm/build-config.sh b/.buildkite/scripts/rocm/build-config.sh index ad0179b163..f6c6ff56ae 100644 --- a/.buildkite/scripts/rocm/build-config.sh +++ b/.buildkite/scripts/rocm/build-config.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # Shared Dockerfile selection for the normal AMD CI build chain. # CI_ROCM_DOCKERFILE_BASE and CI_ROCM_DOCKERFILE accept repository-relative paths. -# For Rock, set CI_ROCM_DOCKERFILE_BASE=docker/Dockerfile.rock_base and -# CI_ROCM_DOCKERFILE=docker/Dockerfile.rock. +# For the legacy ROCm 7.2 stack, set CI_ROCM_DOCKERFILE_BASE=docker/Dockerfile.rocm_72_base +# and CI_ROCM_DOCKERFILE=docker/Dockerfile.rocm_72. # Custom recipes must preserve the CI targets and artifact layout. Additional # COPY inputs also need the corresponding *_CONTENT_FILES overrides. diff --git a/.buildkite/scripts/rocm/stack.sh b/.buildkite/scripts/rocm/stack.sh new file mode 100644 index 0000000000..14bfeb48a0 --- /dev/null +++ b/.buildkite/scripts/rocm/stack.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# +# Derive everything the release pipeline needs about a ROCm stack from its base +# Dockerfile, so the stack's image is the single input to every ROCm script. +# +# Usage: source .buildkite/scripts/rocm/stack.sh +# e.g. docker/Dockerfile.rocm_base (ROCm 10.0) or docker/Dockerfile.rocm_72_base +# +# Exports: +# ROCM_STACK_BASE_DOCKERFILE the base Dockerfile +# ROCM_STACK_DOCKERFILE the matching vLLM Dockerfile (base name minus _base) +# ROCM_STACK_THEROCK 1 if the ROCm SDK comes from TheRock wheels +# ROCM_STACK_VERSION ROCm version, e.g. 10.0.0 or 7.2.3 +# ROCM_STACK_VARIANT wheel variant and Docker Hub tag flavor from major.minor, +# same rule as setup.py (rocm100, rocm72) +# ROCM_STACK_IMAGE_KEY hash of the base Dockerfile (base image cache key) +# ROCM_STACK_WHEEL_KEY hash of the base Dockerfile plus files it COPYs from +# the repo (base wheel cache key) +# ROCM_STACK_ECR_BASE ECR tag of the cached base image +# ROCM_STACK_ECR_IMAGE ECR tag of this commit's release image +# ROCM_STACK_BASE_WHEELS_DIR local dir for base wheels (also the artifact path) +# ROCM_STACK_VLLM_WHEEL_DIR local dir for the vLLM wheel (also the artifact path) + +rocm_stack_init() { + local base="$1" + if [[ ! -f "$base" ]]; then + echo "ROCm base Dockerfile not found: $base" >&2 + return 2 + fi + ROCM_STACK_BASE_DOCKERFILE="$base" + ROCM_STACK_DOCKERFILE="${base%_base}" + + local sdk + sdk=$(sed -nE 's/^ARG ROCM_SDK_VERSION=([0-9.]+).*/\1/p' "$base" | head -1) + if [[ -n "$sdk" ]]; then + ROCM_STACK_THEROCK=1 + ROCM_STACK_VERSION="$sdk" + else + ROCM_STACK_THEROCK=0 + # BASE_IMAGE format: rocm/dev-ubuntu-22.04:7.2.3-complete -> 7.2.3 + ROCM_STACK_VERSION=$(sed -nE 's/^ARG BASE_IMAGE=.*:([0-9]+(\.[0-9]+)+).*/\1/p' "$base" | head -1) + fi + if [[ -z "$ROCM_STACK_VERSION" ]]; then + echo "Could not determine the ROCm version from $base" >&2 + return 2 + fi + + local major minor + IFS=. read -r major minor _ <<< "$ROCM_STACK_VERSION" + ROCM_STACK_VARIANT="rocm${major}${minor}" + + ROCM_STACK_IMAGE_KEY=$(sha256sum "$base" | cut -c1-16) + local inputs + inputs=$(grep -E '^COPY ' "$base" | grep -v -- '--from' | awk '{print $2}' || true) + if [[ -z "$inputs" ]]; then + ROCM_STACK_WHEEL_KEY="$ROCM_STACK_IMAGE_KEY" + else + # shellcheck disable=SC2086 + ROCM_STACK_WHEEL_KEY=$(sha256sum "$base" $inputs | sha256sum | cut -c1-16) + fi + + local ecr="public.ecr.aws/q9t5s3a7/vllm-release-repo" + ROCM_STACK_ECR_BASE="${ecr}:${ROCM_STACK_IMAGE_KEY}-${ROCM_STACK_VARIANT}-base" + ROCM_STACK_ECR_IMAGE="${ecr}:${BUILDKITE_COMMIT:-unknown}-${ROCM_STACK_VARIANT}" + ROCM_STACK_BASE_WHEELS_DIR="artifacts/${ROCM_STACK_VARIANT}-base-wheels" + ROCM_STACK_VLLM_WHEEL_DIR="artifacts/${ROCM_STACK_VARIANT}-vllm-wheel" + + export ROCM_STACK_BASE_DOCKERFILE ROCM_STACK_DOCKERFILE ROCM_STACK_THEROCK \ + ROCM_STACK_VERSION ROCM_STACK_VARIANT ROCM_STACK_IMAGE_KEY \ + ROCM_STACK_WHEEL_KEY ROCM_STACK_ECR_BASE ROCM_STACK_ECR_IMAGE \ + ROCM_STACK_BASE_WHEELS_DIR ROCM_STACK_VLLM_WHEEL_DIR +} + +if [[ $# -gt 0 ]]; then + rocm_stack_init "$1" +fi diff --git a/.buildkite/scripts/upload-rocm-wheels.sh b/.buildkite/scripts/upload-rocm-wheels.sh index 65fca944c7..5cf32bcfb3 100755 --- a/.buildkite/scripts/upload-rocm-wheels.sh +++ b/.buildkite/scripts/upload-rocm-wheels.sh @@ -4,21 +4,31 @@ # # Upload ROCm wheels to S3 with proper index generation # +# Usage: upload-rocm-wheels.sh +# base-dockerfile the ROCm stack, e.g. docker/Dockerfile.rocm_base (see rocm/stack.sh) +# # Required environment variables: # AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY (or IAM role) # S3_BUCKET (default: vllm-wheels) # # S3 path structure: -# s3://vllm-wheels/rocm/{commit}/ - All wheels for this commit -# s3://vllm-wheels/rocm/nightly/ - Index pointing to latest nightly -# s3://vllm-wheels/rocm/{version}/ - Index for release versions +# s3://vllm-wheels/rocm/{commit}/-wheels/ - This stack's wheels for this commit +# s3://vllm-wheels/rocm/{commit}// - Index per ROCm variant (e.g. rocm100) +# s3://vllm-wheels/rocm/nightly// - Index pointing to latest nightly +# s3://vllm-wheels/rocm/{version}// - Index for release versions +# The top-level index.html at each prefix lists every variant present, since +# several ROCm stacks publish into the same prefixes. set -ex # ======== Configuration ======== BUCKET="${S3_BUCKET:-vllm-wheels}" +# shellcheck source=.buildkite/scripts/rocm/stack.sh +source "$(dirname "${BASH_SOURCE[0]}")/rocm/stack.sh" "${1:?Usage: $0 }" ROCM_SUBPATH="rocm/${BUILDKITE_COMMIT}" -S3_COMMIT_PREFIX="s3://$BUCKET/$ROCM_SUBPATH/" +WHEEL_DIRS="$ROCM_STACK_BASE_WHEELS_DIR $ROCM_STACK_VLLM_WHEEL_DIR" +WHEEL_SUBPATH="$ROCM_SUBPATH/${ROCM_STACK_VARIANT}-wheels" +S3_WHEEL_PREFIX="s3://$BUCKET/$WHEEL_SUBPATH/" INDICES_OUTPUT_DIR="rocm-indices" echo "========================================" @@ -49,9 +59,11 @@ source .buildkite/scripts/lib/manylinux.sh # ======== Part 1: Collect and prepare wheels ======== # Collect all wheels +rm -rf all-rocm-wheels "$INDICES_OUTPUT_DIR" mkdir -p all-rocm-wheels -cp artifacts/rocm-base-wheels/*.whl all-rocm-wheels/ 2>/dev/null || true -cp artifacts/rocm-vllm-wheel/*.whl all-rocm-wheels/ 2>/dev/null || true +for dir in $WHEEL_DIRS; do + cp "$dir"/*.whl all-rocm-wheels/ 2>/dev/null || true +done WHEEL_COUNT=$(find all-rocm-wheels -maxdepth 1 -name '*.whl' 2>/dev/null | wc -l) echo "Total wheels to upload: $WHEEL_COUNT" @@ -84,9 +96,9 @@ ls -lh all-rocm-wheels/ # ======== Part 2: Upload wheels to S3 ======== echo "" -echo "Uploading wheels to $S3_COMMIT_PREFIX" +echo "Uploading wheels to $S3_WHEEL_PREFIX" for wheel in all-rocm-wheels/*.whl; do - aws s3 cp "$wheel" "$S3_COMMIT_PREFIX" + aws s3 cp "$wheel" "$S3_WHEEL_PREFIX" done # ======== Part 3: Generate and upload indices ======== @@ -95,28 +107,63 @@ done echo "" echo "Generating indices..." obj_json="rocm-objects.json" -aws s3api list-objects-v2 --bucket "$BUCKET" --prefix "$ROCM_SUBPATH/" --delimiter / --output json > "$obj_json" +aws s3api list-objects-v2 --bucket "$BUCKET" --prefix "$WHEEL_SUBPATH/" --delimiter / --output json > "$obj_json" mkdir -p "$INDICES_OUTPUT_DIR" # Use the existing generate-nightly-index.py # HACK: Replace regex module with stdlib re (same as CUDA script) sed -i 's/import regex as re/import re/g' .buildkite/scripts/generate-nightly-index.py +sed -i 's/import regex as re/import re/g' tools/vllm-rocm/therock_wheels.py +sed -i 's/import pybase64 as base64/import base64/g' tools/vllm-rocm/therock_wheels.py +INDEX_ARGS=(--version "$ROCM_SUBPATH" --wheel-dir "${WHEEL_SUBPATH#rocm/}") +# TheRock's SDK, device kernels and torchvision/torchaudio are linked, not hosted +if [[ "$ROCM_STACK_THEROCK" == "1" ]]; then + $PYTHON tools/vllm-rocm/therock_wheels.py external-links \ + --dockerfile "$ROCM_STACK_BASE_DOCKERFILE" \ + --torch-wheel "$(ls "$ROCM_STACK_BASE_WHEELS_DIR"/torch-*.whl)" > therock-external-links.txt + cat therock-external-links.txt + INDEX_ARGS+=(--external-links therock-external-links.txt) +fi $PYTHON .buildkite/scripts/generate-nightly-index.py \ - --version "$ROCM_SUBPATH" \ + "${INDEX_ARGS[@]}" \ --current-objects "$obj_json" \ --output-dir "$INDICES_OUTPUT_DIR" \ --comment "ROCm commit $BUILDKITE_COMMIT" -# Upload indices to commit directory -echo "Uploading indices to $S3_COMMIT_PREFIX" -aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "$S3_COMMIT_PREFIX" +# Upload this build's variant indices under an S3 prefix (e.g. rocm/nightly), then +# rebuild that prefix's top-level project list from every variant now present. +publish_indices() { + local prefix="$1" + local listing variants merged + echo "Uploading indices to s3://$BUCKET/$prefix/" + aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/$prefix/" --exclude "index.html" + listing=$(aws s3api list-objects-v2 --bucket "$BUCKET" --prefix "$prefix/" \ + --delimiter / --query 'CommonPrefixes[].Prefix' --output text) + variants=$(tr '\t' '\n' <<< "$listing" | sed -n "s|^$prefix/\(rocm[0-9]*\)/$|\1|p" | sort -u) + merged=$(mktemp) + { + echo '' + echo '' + echo ' ' + echo ' ' + for variant in $variants; do + echo " $variant/
" + done + echo ' ' + echo '' + } > "$merged" + echo "Variants under $prefix/: $(echo "$variants" | tr '\n' ' ')" + aws s3 cp "$merged" "s3://$BUCKET/$prefix/index.html" + rm -f "$merged" +} + +publish_indices "$ROCM_SUBPATH" # Only scheduled nightly builds should update the moving nightly index. if [[ "${NIGHTLY:-0}" == "1" ]]; then - echo "Updating rocm/nightly/ index..." - aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/rocm/nightly/" + publish_indices "rocm/nightly" fi # Extract version from vLLM wheel and update version-specific index @@ -129,8 +176,7 @@ if [ -n "$VLLM_WHEEL" ]; then echo "Pure version: $PURE_VERSION" if [[ "$VERSION" != *"dev"* ]]; then - echo "Updating rocm/$PURE_VERSION/ index..." - aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/rocm/$PURE_VERSION/" + publish_indices "rocm/$PURE_VERSION" fi fi @@ -142,14 +188,14 @@ echo "ROCm Wheel Upload Complete!" echo "========================================" echo "" echo "Wheels available at:" -echo " s3://$BUCKET/$ROCM_SUBPATH/" +echo " s3://$BUCKET/$WHEEL_SUBPATH/" echo "" echo "Install command (by commit):" -echo " pip install vllm --extra-index-url https://${BUCKET}.s3.amazonaws.com/$ROCM_SUBPATH/" +echo " pip install vllm --extra-index-url https://${BUCKET}.s3.amazonaws.com/$ROCM_SUBPATH/$ROCM_STACK_VARIANT/" echo "" if [[ "${NIGHTLY:-0}" == "1" ]]; then echo "Install command (nightly):" - echo " pip install vllm --extra-index-url https://${BUCKET}.s3.amazonaws.com/rocm/nightly/" + echo " pip install vllm --extra-index-url https://${BUCKET}.s3.amazonaws.com/rocm/nightly/$ROCM_STACK_VARIANT/" fi echo "" echo "Wheel count: $WHEEL_COUNT" diff --git a/docker/Dockerfile.rock_base b/docker/Dockerfile.rock_base deleted file mode 100644 index 57bf6ae6fe..0000000000 --- a/docker/Dockerfile.rock_base +++ /dev/null @@ -1,453 +0,0 @@ -ARG BASE_IMAGE=ubuntu:22.04 -ARG PYTHON_VERSION=3.12 -ARG TORCH_VERSION=2.12.0+rocm10.0.0 -ARG TORCHVISION_VERSION=0.27.0+rocm10.0.0 -ARG TORCHAUDIO_VERSION=2.11.0+rocm10.0.0 -ARG TRITON_BRANCH="669b31ac" # release/internal/3.8.x as of 09/29/2026 -ARG TRITON_VERSION=3.8.0+git669b31ac -ARG TRITON_REPO="https://github.com/ROCm/triton.git" -ARG FA_BRANCH="0e60e394" -ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git" -ARG AITER_BRANCH="v0.1.25" -ARG AITER_REPO="https://github.com/ROCm/aiter.git" -ARG MORI_BRANCH="v1.2.3.post1" -ARG MORI_REPO="https://github.com/ROCm/mori.git" -ARG ROCM_SYSTEMS_REPO="https://github.com/ROCm/rocm-systems.git" -ARG ROCM_RUNTIME_COMMIT="25e14349f2606a6b95c4ccd45eb92a1835646cf2" - - -# TheRock configuration -ARG SITE_PACKAGES=/usr/local/lib/python${PYTHON_VERSION}/dist-packages -ARG ROCM_RELEASE_WHEELS_MULTIARCH_URL="https://stable.repo.amd.com/rocm/whl-next/" # For pre-built GPU images, packages, etc. -ARG ROCM_NIGHTLY_WHEELS_MULTIARCH_URL="https://nightly.repo.amd.com/rocm/whl-next/" # For torch, triton, etc. wheels with latest fixes. -ARG ROCM_SDK_VERSION=10.0.0 - -# Build configuration -ARG PYTORCH_ROCM_ARCH="gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1200;gfx1201;gfx1150;gfx1151" -ARG AITER_ROCM_ARCH="gfx942;gfx950" - -# Sccache configuration (only used in release pipeline) -ARG USE_SCCACHE -ARG SCCACHE_DOWNLOAD_URL -ARG SCCACHE_ENDPOINT -ARG SCCACHE_BUCKET_NAME=vllm-build-sccache -ARG SCCACHE_REGION_NAME=us-west-2 -ARG SCCACHE_S3_NO_CREDENTIALS=0 - -FROM ${BASE_IMAGE} AS base - -ARG PYTORCH_ROCM_ARCH -ENV PYTORCH_ROCM_ARCH="${PYTORCH_ROCM_ARCH}" -ARG AITER_ROCM_ARCH -ENV AITER_ROCM_ARCH="${AITER_ROCM_ARCH}" - -ARG SITE_PACKAGES -ENV SITE_PACKAGES=${SITE_PACKAGES} -ENV SDK_DEV="$SITE_PACKAGES/_rocm_sdk_devel" \ - SDK_CORE="$SITE_PACKAGES/_rocm_sdk_core" -ENV ROCM_PATH="$SDK_DEV" \ - ROCM_HOME="$SDK_DEV" \ - HIP_PATH="$SDK_DEV" \ - HIP_CLANG_PATH="$SDK_DEV/lib/llvm/bin" \ - HIP_DEVICE_LIB_PATH="$SDK_CORE/lib/llvm/amdgcn/bitcode" -ENV LD_LIBRARY_PATH="$SDK_DEV/lib:$SDK_DEV/lib/rocm_sysdeps/lib:$SDK_CORE/lib:$SDK_CORE/lib/rocm_sysdeps/lib" -ENV PATH="$SDK_DEV/bin:$SDK_DEV/lib/llvm/bin:$SDK_CORE/bin:$PATH" - -# Required for RCCL in ROCm7.1 -ENV HSA_NO_SCRATCH_RECLAIM=1 - -ARG PYTHON_VERSION -ENV PYTHON_VERSION=${PYTHON_VERSION} - -RUN mkdir -p /app -WORKDIR /app -ENV DEBIAN_FRONTEND=noninteractive - -# Install Python and other dependencies -RUN apt-get update -y \ - && apt-get install -y software-properties-common build-essential git curl sudo vim less libgfortran5 libopenmpi-dev libpci-dev liblzma-dev pkg-config libdrm-dev libnuma-dev kmod \ - && for i in 1 2 3; do \ - add-apt-repository -y ppa:deadsnakes/ppa && break || \ - { echo "Attempt $i failed, retrying in 5s..."; sleep 5; }; \ - done \ - && apt-get update -y \ - && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv \ - python${PYTHON_VERSION}-lib2to3 python-is-python3 \ - && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \ - && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \ - && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \ - && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \ - && python3 --version && python3 -m pip --version - -RUN pip install -U packaging 'cmake<4' ninja wheel 'setuptools<80' pybind11==3.0.4 Cython -RUN apt-get update && apt-get install -y libjpeg-dev libsox-dev libsox-fmt-all sox && rm -rf /var/lib/apt/lists/* - -# Install ROCm SDK via TheRock -ARG ROCM_RELEASE_WHEELS_MULTIARCH_URL -ARG ROCM_NIGHTLY_WHEELS_MULTIARCH_URL -ARG ROCM_SDK_VERSION -RUN pip install --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \ - --extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \ - "rocm[libraries,devel]${ROCM_SDK_VERSION:+==${ROCM_SDK_VERSION}}" \ - && rocm-sdk init - -# Install PyTorch wheels (nightly for latest torch, release as fallback) -ARG TORCH_VERSION -ARG TORCHAUDIO_VERSION -ARG TORCHVISION_VERSION -RUN pip install \ - --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \ - --extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \ - "torch==${TORCH_VERSION}" \ - "torchaudio==${TORCHAUDIO_VERSION}" \ - "torchvision==${TORCHVISION_VERSION}" - -# Install the AMD-compiled device kernels corresponding to PYTORCH_ROCM_ARCH: - -# Possible values that can be in PYTORCH_ROCM_ARCH: -# -# gfx90a (MI210/250), gfx942 (MI300), gfx950 (MI350), gfx1100 (7900XTX), -# gfx1101 (7800XT), gfx1200 (9070XT), gfx1201 (9070), gfx1150/1151 (Strix APU) - -# For each arch in PYTORCH_ROCM_ARCH, install the precompiled GPU code that -# PyTorch needs to run on that specific chip. -SHELL ["/bin/bash", "-c"] -RUN TORCH_VER=$(python3 -c "import torch; print(torch.__version__)") \ - && for arch in $(echo "${PYTORCH_ROCM_ARCH}" | tr ';' ' '); do \ - echo "Installing device packages for ${arch}" \ - && pip install --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \ - --extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \ - amd-torch-device-${arch}==${TORCH_VER} \ - rocm-sdk-device-${arch}==${ROCM_SDK_VERSION}; \ - done - -# Remove gfx90a xnack kpacks (ASAN artifacts that cause hipErrorInvalidImage on MI250) -RUN rm -f "${SITE_PACKAGES}/torch/.kpack/torch_gfx90a:xnack+.kpack" \ - "${SITE_PACKAGES}/torch/.kpack/torch_gfx90a:xnack-.kpack" - -RUN cd $SITE_PACKAGES/_rocm_sdk_core/share/amd_smi && pip install . -ENV PYTHONPATH=$SITE_PACKAGES/_rocm_sdk_core/share/amd_smi - -# TheRock runtime fixes -RUN if [ -f "${SITE_PACKAGES}/rocm_sdk/__init__.py" ]; then \ - sed -i 's/rtld_global: bool = True/rtld_global: bool = False/g' "${SITE_PACKAGES}/rocm_sdk/__init__.py"; \ - fi \ - && if [ -f "${SITE_PACKAGES}/_rocm_sdk_core/lib/librocm_smi64.so.1" ] \ - && [ ! -e "${ROCM_PATH}/lib/librocm_smi64.so.7" ]; then \ - ln -sfn "${SITE_PACKAGES}/_rocm_sdk_core/lib/librocm_smi64.so.1" "${ROCM_PATH}/lib/librocm_smi64.so.7"; \ - fi \ - && printf '%s\n%s\n' "${ROCM_PATH}/lib" "${SITE_PACKAGES}/_rocm_sdk_core/lib" > /etc/ld.so.conf.d/rocm-pip.conf \ - && for f in ${SITE_PACKAGES}/_rocm_sdk_core/lib/libamdhip64.so*; do \ - bn=$(basename "$f"); \ - if [ -f "${ROCM_PATH}/lib/${bn}" ] && [ -f "$f" ] && [ ! -L "$f" ]; then \ - rm -f "$f"; \ - ln -sfn "${ROCM_PATH}/lib/${bn}" "$f"; \ - fi; \ - done \ - && for f in librocprofiler-sdk.so.1 librocprofiler-register.so.0 \ - librocprofiler-sdk-roctx.so.1; do \ - if [ -e "${SITE_PACKAGES}/_rocm_sdk_core/lib/${f}" ] \ - && [ -e "${SITE_PACKAGES}/_rocm_sdk_devel/lib/${f}" ]; then \ - rm -f "${SITE_PACKAGES}/_rocm_sdk_core/lib/${f}"; \ - ln -sfn "${SITE_PACKAGES}/_rocm_sdk_devel/lib/${f}" \ - "${SITE_PACKAGES}/_rocm_sdk_core/lib/${f}"; \ - fi; \ - done \ - && ldconfig - -# Install sccache if USE_SCCACHE is enabled (for release builds) -ARG USE_SCCACHE -ARG SCCACHE_DOWNLOAD_URL -ARG SCCACHE_ENDPOINT -ARG SCCACHE_BUCKET_NAME -ARG SCCACHE_REGION_NAME -ARG SCCACHE_S3_NO_CREDENTIALS -RUN if [ "$USE_SCCACHE" = "1" ]; then \ - echo "Installing sccache..." \ - && SCCACHE_ARCH="x86_64" \ - && SCCACHE_VERSION="v0.8.1" \ - && SCCACHE_DL_URL="${SCCACHE_DOWNLOAD_URL:-https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}-unknown-linux-musl.tar.gz}" \ - && curl -L -o /tmp/sccache.tar.gz ${SCCACHE_DL_URL} \ - && tar -xzf /tmp/sccache.tar.gz -C /tmp \ - && mv /tmp/sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}-unknown-linux-musl/sccache /usr/bin/sccache \ - && chmod +x /usr/bin/sccache \ - && rm -rf /tmp/sccache.tar.gz /tmp/sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}-unknown-linux-musl \ - && sccache --version; \ - fi - -# Setup sccache for HIP compilation via HIP_CLANG_PATH -# This creates wrapper scripts in a separate directory and points HIP to use them -# This avoids modifying the original ROCm binaries which can break detection -# NOTE: HIP_CLANG_PATH is NOT set as ENV to avoid affecting downstream images (Dockerfile.rocm) -# Instead, each build stage should export HIP_CLANG_PATH=/opt/sccache-wrappers if USE_SCCACHE=1 -RUN if [ "$USE_SCCACHE" = "1" ]; then \ - echo "Setting up sccache wrappers for HIP compilation..." \ - && mkdir -p /opt/sccache-wrappers \ - && printf '#!/bin/bash\nexec sccache %s/lib/llvm/bin/clang++ "$@"\n' "${ROCM_PATH}" > /opt/sccache-wrappers/clang++ \ - && chmod +x /opt/sccache-wrappers/clang++ \ - && printf '#!/bin/bash\nexec sccache %s/lib/llvm/bin/clang "$@"\n' "${ROCM_PATH}" > /opt/sccache-wrappers/clang \ - && chmod +x /opt/sccache-wrappers/clang \ - && echo "sccache wrappers created in /opt/sccache-wrappers"; \ - fi - -ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET_NAME}} -ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}} -ENV SCCACHE_S3_NO_CREDENTIALS=${USE_SCCACHE:+${SCCACHE_S3_NO_CREDENTIALS}} -ENV SCCACHE_IDLE_TIMEOUT=${USE_SCCACHE:+0} - - -### -### MORI Build -### -FROM base AS build_mori -ARG MORI_BRANCH -ARG MORI_REPO -RUN git clone ${MORI_REPO} -RUN cd mori \ - && git checkout ${MORI_BRANCH} \ - && git submodule update --init --recursive \ - && ln -sfn ${ROCM_PATH} /opt/rocm \ - && pip install "setuptools_scm>=6.2" \ - && CMAKE_PREFIX_PATH="${ROCM_PATH}:${ROCM_PATH}/lib/rocm_sysdeps" \ - python3 setup.py bdist_wheel --dist-dir=dist && ls /app/mori/dist/*.whl -RUN mkdir -p /app/install && cp /app/mori/dist/*.whl /app/install - -### -### FlashAttention Build -### -FROM base AS build_fa -ARG FA_BRANCH -ARG FA_REPO -ARG USE_SCCACHE -RUN git clone ${FA_REPO} -RUN cd flash-attention \ - && git checkout ${FA_BRANCH} \ - && git submodule update --init \ - && if [ "$USE_SCCACHE" = "1" ]; then \ - export HIP_CLANG_PATH=/opt/sccache-wrappers \ - && sccache --show-stats; \ - fi \ - && GPU_ARCHS=$(echo ${PYTORCH_ROCM_ARCH} | sed -e 's/;gfx1[0-9]\{3\}//g') python3 setup.py bdist_wheel --dist-dir=dist \ - && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi -RUN mkdir -p /app/install && cp /app/flash-attention/dist/*.whl /app/install - - -### -### AITER Build -### -FROM base AS build_aiter -ARG AITER_BRANCH -ARG AITER_REPO -ARG USE_SCCACHE -RUN git clone --recursive --branch ${AITER_BRANCH} ${AITER_REPO} -RUN cd aiter \ - && git submodule update --init --recursive \ - && pip install -r requirements.txt -# AITER_USE_SYSTEM_TRITON keeps setup.py from resolving its own triton, which would -# displace the pinned one already installed in base. -RUN pip install pyyaml && cd aiter \ - && if [ "$USE_SCCACHE" = "1" ]; then \ - export HIP_CLANG_PATH=/opt/sccache-wrappers \ - && sccache --show-stats; \ - fi \ - && PREBUILD_KERNELS=1 AITER_USE_SYSTEM_TRITON=1 GPU_ARCHS=${AITER_ROCM_ARCH} python3 setup.py bdist_wheel --dist-dir=dist \ - && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi \ - && ls /app/aiter/dist/*.whl -RUN mkdir -p /app/install && cp /app/aiter/dist/*.whl /app/install - - -### -### Triton Build -### -FROM base AS build_triton -ARG TRITON_BRANCH -ARG TRITON_REPO -RUN git clone ${TRITON_REPO} -RUN cd triton \ - && git checkout ${TRITON_BRANCH} \ - && if [ ! -f setup.py ]; then cd python; fi \ - && python3 setup.py bdist_wheel --dist-dir=dist \ - && mkdir -p /app/install && cp dist/*.whl /app/install - - -### -### ROCr + CLR build -### -# Brings in multi-stream perf improvements; drop after these arrive in an official release -FROM base AS build_rocm_runtime -ARG ROCM_SYSTEMS_REPO -ARG ROCM_RUNTIME_COMMIT -# libgl-dev: rocclr does find_package(OpenGL REQUIRED) unconditionally and the wheel image has -# no GL headers. The link must not pick up a libGL DT_NEEDED, which is asserted after the build. -RUN apt-get update \ - && apt-get install -y --no-install-recommends g++ libdw-dev xxd libgl-dev libglx-dev \ - && rm -rf /var/lib/apt/lists/* -RUN pip install --no-cache-dir CppHeaderParser -RUN git init -q /src && cd /src \ - && git remote add origin ${ROCM_SYSTEMS_REPO} \ - && git config core.sparseCheckout true \ - && git config remote.origin.promisor true \ - && git config remote.origin.partialclonefilter blob:none \ - && git sparse-checkout init --cone \ - && git sparse-checkout set projects/rocr-runtime projects/clr projects/hip shared cmake \ - && git fetch --filter=blob:none --depth 1 origin ${ROCM_RUNTIME_COMMIT} \ - && git checkout -q FETCH_HEAD -# libelf is taken from the SDK's own vendored sysdeps rather than apt -- the wheel image has no -# /usr/include/libelf.h -- so ROCr links against the copy the wheel itself was built against. -ENV SYSDEPS="$SDK_DEV/lib/rocm_sysdeps" -# ROCr first, and installed into $SDK_DEV, because the clr build below needs its headers. -RUN cd /src/projects/rocr-runtime \ - && cmake -B build -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH="$SDK_DEV;$SYSDEPS" \ - -DCMAKE_INSTALL_PREFIX="$SDK_DEV" \ - -DLIBELF_INCLUDE_DIRS="$SYSDEPS/include" \ - -DLIBELF_LIBRARIES="$SYSDEPS/lib/libelf.so" \ - && cmake --build build --parallel 16 \ - && cmake --install build \ - && cmake --install build --prefix /rocr-install --strip -RUN cd /src/projects/clr \ - && cmake -B build -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DCLR_BUILD_HIP=ON \ - -DCLR_BUILD_OCL=OFF \ - -DROCM_KPACK_ENABLED=ON \ - -DHIP_COMMON_DIR=/src/projects/hip \ - -DROCM_PATH="$SDK_DEV" \ - -DCMAKE_PREFIX_PATH="$SDK_DEV;$SYSDEPS" \ - -DCMAKE_INSTALL_PREFIX="$SDK_DEV" \ - -DHIP_LLVM_ROOT="$SDK_DEV/lib/llvm" \ - && cmake --build build --parallel 16 \ - && cmake --install build --prefix /clr-install --strip -RUN mkdir -p /staging/lib \ - && cp -P /rocr-install/lib/libhsa-runtime64.so* /staging/lib/ \ - && cp -P /clr-install/lib/libamdhip64.so* /staging/lib/ \ - && ls -l /staging/lib - - -### -### Final Build -### - -# Wheel release stage - -# only includes dependencies used by wheel release pipeline -FROM base AS debs_wheel_release -RUN mkdir /app/debs -RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs -RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs -RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs - -FROM base AS debs -RUN mkdir /app/debs -RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs -RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs -RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs -RUN --mount=type=bind,from=build_mori,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs - -FROM base AS final -ARG TRITON_VERSION - -# torch 2.12 implicitly attaches rocprofiler-sdk; its queue interposition hangs at HSA -# teardown, so disable it. TODO: drop once ROCm's stock rocprofiler carries the fix. -ENV ROCPROFILER_QUEUE_INTERPOSITION=0 - -COPY --from=build_rocm_runtime /staging/lib/ ${SDK_DEV}/lib/ -# Hard link the patched library onto every stock file name, in both SDK trees, so that the -# SONAME lookup and the hard coded paths all resolve to it. Hard links rather than symlinks -# because ldconfig rewrites SONAME symlinks but cannot re-point an inode. $SDK_CORE keeps its -# own libhsa-runtime64 copy, which is on LD_LIBRARY_PATH and in ld.so.conf; relinking it is -# belt and braces rather than strictly required, but it leaves no stock ROCr in the image. -RUN set -eux; \ - cd "$SDK_DEV/lib"; \ - ours_hsa="$(readlink -f libhsa-runtime64.so.1)"; \ - ours_hip="$(readlink -f libamdhip64.so.7)"; \ - for f in "$SDK_DEV"/lib/libhsa-runtime64.so.1* "$SDK_CORE"/lib/libhsa-runtime64.so.1* \ - "$SDK_DEV"/lib/libamdhip64.so.7* "$SDK_CORE"/lib/libamdhip64.so.7*; do \ - [ -f "$f" ] && [ ! -L "$f" ] || continue; \ - case "${f##*/}" in libhsa*) ours="$ours_hsa";; *) ours="$ours_hip";; esac; \ - [ "$f" -ef "$ours" ] || ln -f "$ours" "$f"; \ - done - -RUN ldconfig - -# This is to prevent resolvers from checking METADATA and RECORD and clobbering -# the Triton source build we installed. Constraints files are unable to help -# with this. However, these entries are optional and can be blanked which -# solves the problem. -RUN TORCH_DIST_INFO=$(ls -d ${SITE_PACKAGES}/torch-*.dist-info) \ - && grep -q '^Requires-Dist: triton' "$TORCH_DIST_INFO/METADATA" \ - && sed -i '/^Requires-Dist: triton/d' "$TORCH_DIST_INFO/METADATA" \ - && sed -i "s|^\(${TORCH_DIST_INFO##*/}/METADATA\),.*|\1,,|" "$TORCH_DIST_INFO/RECORD" - -RUN --mount=type=bind,from=debs,src=/app/debs,target=/install \ - pip install /install/*.whl \ - && ! pip check 2>&1 | grep -iE '^(torch|triton) ' \ - && test "$(pip show triton | awk '/^Version:/ {print $2}')" = "${TRITON_VERSION}" \ - # Create a constraints file to protect the versions we need, especially - # triton, requirements will try to reinstall repeatedly. - && pip list --format=freeze \ - | grep -E '^(torch|triton|torchaudio|torchvision|pybind11)==' > /etc/rocm-constraints.txt \ - && echo "ROCm constraints:" && cat /etc/rocm-constraints.txt \ - && test "$(wc -l < /etc/rocm-constraints.txt)" -eq 5 \ - && grep -q '^pybind11==3\.0\.' /etc/rocm-constraints.txt \ - && python3 -c "import torch; assert torch.version.hip, f'non-ROCm torch: {torch.__version__}'; print(f'torch={torch.__version__}')" \ - && python3 -c "import triton; print(f'triton={triton.__version__}')" \ - && pip list 2>/dev/null | grep -iE '^(torch|triton) ' - -ARG BASE_IMAGE -ARG PYTHON_VERSION -ARG ROCM_SDK_VERSION -ARG TORCH_VERSION -ARG TORCHAUDIO_VERSION -ARG TORCHVISION_VERSION -ARG PYTORCH_ROCM_ARCH -ARG AITER_ROCM_ARCH -ARG TRITON_BRANCH -ARG TRITON_REPO -ARG FA_BRANCH -ARG FA_REPO -ARG AITER_BRANCH -ARG AITER_REPO -ARG MORI_BRANCH -ARG MORI_REPO -ARG ROCM_SYSTEMS_REPO -ARG ROCM_RUNTIME_COMMIT -ARG TRITON_VERSION -# Record hashes for Triton and pybind11 so we can verify later. -RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \ - && echo "PYTHON_VERSION: ${PYTHON_VERSION}" >> /app/versions.txt \ - && echo "ROCM_SDK_VERSION: ${ROCM_SDK_VERSION}" >> /app/versions.txt \ - && echo "TORCH_VERSION: ${TORCH_VERSION}" >> /app/versions.txt \ - && echo "TORCHAUDIO_VERSION: ${TORCHAUDIO_VERSION}" >> /app/versions.txt \ - && echo "TORCHVISION_VERSION: ${TORCHVISION_VERSION}" >> /app/versions.txt \ - && echo "PYTORCH_ROCM_ARCH: ${PYTORCH_ROCM_ARCH}" >> /app/versions.txt \ - && echo "AITER_ROCM_ARCH: ${AITER_ROCM_ARCH}" >> /app/versions.txt \ - && echo "TRITON_BRANCH: ${TRITON_BRANCH}" >> /app/versions.txt \ - && echo "TRITON_REPO: ${TRITON_REPO}" >> /app/versions.txt \ - && echo "FA_BRANCH: ${FA_BRANCH}" >> /app/versions.txt \ - && echo "FA_REPO: ${FA_REPO}" >> /app/versions.txt \ - && echo "AITER_BRANCH: ${AITER_BRANCH}" >> /app/versions.txt \ - && echo "AITER_REPO: ${AITER_REPO}" >> /app/versions.txt \ - && echo "MORI_BRANCH: ${MORI_BRANCH}" >> /app/versions.txt \ - && echo "MORI_REPO: ${MORI_REPO}" >> /app/versions.txt \ - && echo "ROCM_SYSTEMS_REPO: ${ROCM_SYSTEMS_REPO}" >> /app/versions.txt \ - && echo "ROCM_RUNTIME_COMMIT: ${ROCM_RUNTIME_COMMIT}" >> /app/versions.txt \ - # Fingerprint our triton so Dockerfile.rocm can prove it survived the build. - && echo "TRITON_VERSION: $(pip show triton | awk '/^Version:/ {print $2}')" >> /app/versions.txt \ - && echo "TRITON_SHA256: $(find $(python3 -c 'import triton, os; print(os.path.dirname(triton.__file__))') \ - -type f \( -name '*.py' -o -name '*.so' \) | sort | xargs sha256sum | sha256sum | cut -d' ' -f1)" \ - >> /app/versions.txt \ - && echo "PYBIND11_VERSION: $(pip show pybind11 | awk '/^Version:/ {print $2}')" >> /app/versions.txt \ - && echo "PYBIND11_INTERNALS_VERSION: $(grep -rhE '^#[[:space:]]*define PYBIND11_INTERNALS_VERSION' \ - $(python3 -c 'import pybind11; print(pybind11.get_include())') | awk '{print $NF}')" \ - >> /app/versions.txt \ - && echo "PYBIND11_SHA256: $(find $(python3 -c 'import pybind11; print(pybind11.get_include())') \ - -type f -name '*.h' | sort | xargs sha256sum | sha256sum | cut -d' ' -f1)" \ - >> /app/versions.txt diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm index 36aa0aa4df..029d003b55 100644 --- a/docker/Dockerfile.rocm +++ b/docker/Dockerfile.rocm @@ -280,8 +280,9 @@ COPY --from=fetch_vllm \ ${COMMON_WORKDIR}/vllm/requirements/rocm.txt \ requirements/ +# Use constraints file to protect triton, pybind11, and torch dependencies. RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \ - uv pip install --system -r requirements/rocm.txt + uv pip install --system -c /etc/rocm-constraints.txt -r requirements/rocm.txt # Fetch only the pinned ROCm Triton kernels package. Keeping this before the # vLLM source COPYs makes the download reusable when native sources change. @@ -322,8 +323,9 @@ ENV SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+rocm.csrc.build" RUN --mount=type=bind,source=pyproject.toml,target=${COMMON_WORKDIR}/vllm/pyproject.toml \ --mount=type=cache,id=vllm-rocm-ccache,target=/root/.cache/ccache \ - grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \ + { grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \ cmake/external_projects/triton_kernels.cmake \ + || echo "WARNING: Dockerfile.rocm ROCM_TRITON_KERNELS_COMMIT (${ROCM_TRITON_KERNELS_COMMIT}) does not match cmake TRITON_KERNELS_TAG; the Docker build uses the Dockerfile pin, cmake governs only non-Docker builds"; } \ && export CCACHE_BASEDIR="$PWD" \ && echo "=== ccache stats before ROCm native build ===" \ && (ccache --show-stats || true) \ @@ -366,7 +368,8 @@ COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/requirements /requirements COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tests /tests COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/examples /examples -COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools /tools +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools/benchmark_helion_kernels.py /tools/benchmark_helion_kernels.py +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile /docker/Dockerfile COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.cpu /docker/Dockerfile.cpu COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/ @@ -386,7 +389,6 @@ ARG NIXL_BRANCH="1683cf3b7f3d11674c03c5e861cea22339876c96" ARG NIXL_REPO="https://github.com/ai-dynamo/nixl.git" ARG UCX_BRANCH="96e58a16039f6d7d213bc967b8069238742c5194" ARG UCX_REPO="https://github.com/openucx/ucx.git" -ENV ROCM_PATH=/opt/rocm ENV UCX_HOME=/usr/local/ucx ENV NIXL_HOME=/usr/local/nixl @@ -541,7 +543,7 @@ RUN --mount=type=tmpfs,target=/tmp/lmcache-wheel \ c_ops="$(find /tmp/lmcache-wheel -type f -name 'c_ops*.so')"; \ test "$(find /tmp/lmcache-wheel -type f -name 'c_ops*.so' | wc -l)" -eq 1; \ expected="$(printf '%s' "${LMCACHE_ROCM_ARCH}" | tr ';' '\n' | sort -u)"; \ - actual="$(/opt/rocm/llvm/bin/llvm-objdump --offloading "${c_ops}" \ + actual="$(llvm-objdump --offloading "${c_ops}" \ | grep -oE 'gfx[0-9a-z]+' | sort -u)"; \ if [ "${actual}" != "${expected}" ]; then \ printf 'LMCache code objects mismatch\nexpected:\n%s\nactual:\n%s\n' \ @@ -549,32 +551,10 @@ RUN --mount=type=tmpfs,target=/tmp/lmcache-wheel \ exit 1; \ fi -# ROCShmem build stage - split from DeepEP so changing DEEPEP_BRANCH does not -# invalidate the slow ROCShmem build. -FROM base AS build_rocshmem -ARG ROCSHMEM_BRANCH="f0acb0c6" -ARG ROCSHMEM_REPO="https://github.com/ROCm/rocm-systems.git" -# DeepEP only supports gfx942 and gfx950; build ROCShmem for the same set so -# it can be linked against DeepEP without arch mismatches. -ARG DEEPEP_ROCM_ARCH="gfx942;gfx950" -ENV ROCM_PATH=/opt/rocm -ENV ROCSHMEM_DIR=/opt/rocshmem - -RUN --mount=type=cache,target=/root/.cache/ccache \ - git clone --no-checkout --filter=blob:none ${ROCSHMEM_REPO} \ - && cd rocm-systems \ - && git sparse-checkout set --cone projects/rocshmem \ - && git checkout ${ROCSHMEM_BRANCH} \ - && mkdir -p projects/rocshmem/build \ - && cd projects/rocshmem/build \ - && CC="ccache gcc" CXX="ccache g++" INSTALL_PREFIX=${ROCSHMEM_DIR} \ - bash ../scripts/build_configs/all_backends \ - -DROCM_PATH=${ROCM_PATH} \ - -DGPU_TARGETS="${DEEPEP_ROCM_ARCH}" \ - -DUSE_EXTERNAL_MPI=OFF - -# DeepEP build stage - depends on ROCShmem, builds the HIP kernel wheel. -FROM build_rocshmem AS build_deepep +# DeepEP build stage. ROCShmem is provided by TheRock SDK +# ROCSHMEM_DIR set to the SDK install path so DeepEP finds the headers, etc. +FROM base AS build_deepep +ENV ROCSHMEM_DIR=${ROCM_PATH} ARG DEEPEP_BRANCH="0f63d3e9" ARG DEEPEP_REPO="https://github.com/ROCm/DeepEP.git" ARG DEEPEP_NIC="cx7" @@ -693,8 +673,9 @@ RUN if [ "$GIT_REPO_CHECK" != "0" ]; then \ # Install the packaging helpers before modifying requirements/rocm.txt. RUN --mount=type=cache,target=/root/.cache/uv \ - grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \ + { grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \ vllm/cmake/external_projects/triton_kernels.cmake \ + || echo "WARNING: Dockerfile.rocm ROCM_TRITON_KERNELS_COMMIT (${ROCM_TRITON_KERNELS_COMMIT}) does not match cmake TRITON_KERNELS_TAG; the Docker build uses the Dockerfile pin, cmake governs only non-Docker builds"; } \ && cd vllm \ && uv pip install --system setuptools_scm regex @@ -750,7 +731,8 @@ COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/requirements /requir COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tests /tests COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/examples /examples -COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools /tools +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools/benchmark_helion_kernels.py /tools/benchmark_helion_kernels.py +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile /docker/Dockerfile COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.cpu /docker/Dockerfile.cpu COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/ @@ -795,9 +777,6 @@ RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \ --mount=type=bind,from=build_deepep,src=/app/deep_install,target=/deep_install \ uv pip install --system /nixl_install/*.whl /deep_install/*.whl -# Copy ROCShmem runtime libraries. -COPY --from=build_rocshmem /opt/rocshmem /opt/rocshmem - # RDMA userspace libraries plus FFmpeg dev libs needed by torchcodec. RUN timeout --kill-after=30s 5m apt-get update -q -y --error-on=any \ -o Acquire::Retries=3 \ @@ -830,7 +809,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \ # Pre-install shared ROCm runtime dependencies. COPY requirements/common.txt requirements/rocm.txt /tmp/ci-base-requirements/ RUN --mount=type=cache,target=/root/.cache/uv \ - uv pip install --system -r /tmp/ci-base-requirements/rocm.txt \ + uv pip install --system -c /etc/rocm-constraints.txt \ + -r /tmp/ci-base-requirements/rocm.txt \ && rm -rf /tmp/ci-base-requirements # Enable fast and less brittle model downloads in tests. @@ -908,8 +888,6 @@ RUN mkdir src && mv vllm src/vllm # Catch GPU<->CPU syncs in execute_model/sample_tokens during tests. ENV VLLM_GPU_SYNC_CHECK=error -# Run the structural smoke test inside BuildKit, where the test image layers -# are already available. Export only a build-scoped success marker. FROM test AS test_smoke ARG ROCM_SMOKE_ID RUN --network=none \ @@ -956,15 +934,37 @@ RUN --mount=type=bind,from=export_vllm,src=/,target=/install \ --mount=type=cache,target=/root/.cache/uv \ cd /install \ && echo "opentelemetry-api<=1.40.0" > /tmp/otel-constraints.txt \ - && uv pip install --system -c /tmp/otel-constraints.txt -r requirements/rocm.txt \ + && uv pip install --system -c /etc/rocm-constraints.txt -c /tmp/otel-constraints.txt -r requirements/rocm.txt \ && pip uninstall -y vllm \ - && uv pip install --system -c /tmp/otel-constraints.txt *.whl \ + && uv pip install --system -c /etc/rocm-constraints.txt -c /tmp/otel-constraints.txt *.whl \ && rm /tmp/otel-constraints.txt # Install NIXL ROCm wheel RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \ uv pip install --system /nixl_install/*.whl +# Recompute the base's triton fingerprint to detect resolver interference. +RUN EXPECTED=$(grep '^TRITON_SHA256: ' /app/versions.txt | cut -d' ' -f2) \ + && ACTUAL=$(find $(python3 -c 'import triton, os; print(os.path.dirname(triton.__file__))') \ + -type f \( -name '*.py' -o -name '*.so' \) | sort | xargs sha256sum | sha256sum | cut -d' ' -f1) \ + && echo "triton sha256 expected=${EXPECTED} actual=${ACTUAL}" \ + && test -n "${EXPECTED}" \ + && test "${EXPECTED}" = "${ACTUAL}" \ + && python3 -c "import torch; assert torch.version.hip, f'non-ROCm torch: {torch.__version__}'; print(f'torch={torch.__version__}')" + +# Recompute the base's pybind11 fingerprint to detect resolver interference. +RUN EXPECTED=$(grep '^PYBIND11_SHA256: ' /app/versions.txt | cut -d' ' -f2) \ + && EXPECTED_ABI=$(grep '^PYBIND11_INTERNALS_VERSION: ' /app/versions.txt | cut -d' ' -f2) \ + && INC=$(python3 -c 'import pybind11; print(pybind11.get_include())') \ + && ACTUAL=$(find "${INC}" -type f -name '*.h' | sort | xargs sha256sum | sha256sum | cut -d' ' -f1) \ + && ACTUAL_ABI=$(grep -rhE '^#[[:space:]]*define PYBIND11_INTERNALS_VERSION' "${INC}" | awk '{print $NF}') \ + && echo "pybind11 $(pip show pybind11 | awk '/^Version:/ {print $2}')" \ + "sha256 expected=${EXPECTED} actual=${ACTUAL}" \ + "internals expected=${EXPECTED_ABI} actual=${ACTUAL_ABI}" \ + && test -n "${EXPECTED}" \ + && test "${EXPECTED}" = "${ACTUAL}" \ + && test "${EXPECTED_ABI}" = "${ACTUAL_ABI}" + ARG COMMON_WORKDIR ARG BASE_IMAGE ARG NIC_BACKEND diff --git a/docker/Dockerfile.rock b/docker/Dockerfile.rocm_72 similarity index 92% rename from docker/Dockerfile.rock rename to docker/Dockerfile.rocm_72 index ed66b8e85e..a24b65458a 100644 --- a/docker/Dockerfile.rock +++ b/docker/Dockerfile.rocm_72 @@ -3,7 +3,7 @@ ARG REMOTE_VLLM="0" ARG VLLM_REPO="https://github.com/vllm-project/vllm.git" ARG VLLM_BRANCH="main" ARG COMMON_WORKDIR=/app -ARG BASE_IMAGE=rocm/vllm-dev:base +ARG BASE_IMAGE=vllm/vllm-openai-rocm:base-nightly-rocm72 ARG CI_BASE_IMAGE=rocm/vllm-dev:ci_base ARG ROCM_TRITON_KERNELS_COMMIT=669b31acc1dd1b3fd93286afd5db67f65d9f7557 ARG INSTALL_LMCACHE=false @@ -46,7 +46,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \ # Note: mold is installed but not set as the system default linker because # some packages use JIT compilation at runtime with flags mold does not support. # Build stages opt in via LDFLAGS="-fuse-ld=mold". -# Remove sccache only if not using sccache (it exists in base image from Dockerfile.rocm_base) +# Remove sccache only if not using sccache (it exists in base image from Dockerfile.rocm_72_base) ARG USE_SCCACHE RUN if [ "$USE_SCCACHE" != "1" ]; then \ apt-get purge -y sccache || true; \ @@ -280,9 +280,8 @@ COPY --from=fetch_vllm \ ${COMMON_WORKDIR}/vllm/requirements/rocm.txt \ requirements/ -# Use constraints file to protect triton, pybind11, and torch dependencies. RUN --mount=type=cache,id=vllm-rocm-uv,target=/root/.cache/uv \ - uv pip install --system -c /etc/rocm-constraints.txt -r requirements/rocm.txt + uv pip install --system -r requirements/rocm.txt # Fetch only the pinned ROCm Triton kernels package. Keeping this before the # vLLM source COPYs makes the download reusable when native sources change. @@ -323,9 +322,8 @@ ENV SETUPTOOLS_SCM_PRETEND_VERSION="0.0.0+rocm.csrc.build" RUN --mount=type=bind,source=pyproject.toml,target=${COMMON_WORKDIR}/vllm/pyproject.toml \ --mount=type=cache,id=vllm-rocm-ccache,target=/root/.cache/ccache \ - { grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \ + grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \ cmake/external_projects/triton_kernels.cmake \ - || echo "WARNING: Dockerfile.rock ROCM_TRITON_KERNELS_COMMIT (${ROCM_TRITON_KERNELS_COMMIT}) does not match cmake TRITON_KERNELS_TAG; the Docker build uses the Dockerfile pin, cmake governs only non-Docker builds"; } \ && export CCACHE_BASEDIR="$PWD" \ && echo "=== ccache stats before ROCm native build ===" \ && (ccache --show-stats || true) \ @@ -368,12 +366,11 @@ COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/requirements /requirements COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tests /tests COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/examples /examples -COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools/benchmark_helion_kernels.py /tools/benchmark_helion_kernels.py -COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools /tools COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile /docker/Dockerfile COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.cpu /docker/Dockerfile.cpu -COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/ -COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm_base /docker/Dockerfile.rocm_base +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm_72 /docker/ +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm_72_base /docker/ COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/ci-rocm.hcl /docker/ci-rocm.hcl COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/docker-bake.hcl /docker/docker-bake.hcl COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/docker-bake-rocm.hcl /docker/docker-bake-rocm.hcl @@ -389,6 +386,7 @@ ARG NIXL_BRANCH="1683cf3b7f3d11674c03c5e861cea22339876c96" ARG NIXL_REPO="https://github.com/ai-dynamo/nixl.git" ARG UCX_BRANCH="96e58a16039f6d7d213bc967b8069238742c5194" ARG UCX_REPO="https://github.com/openucx/ucx.git" +ENV ROCM_PATH=/opt/rocm ENV UCX_HOME=/usr/local/ucx ENV NIXL_HOME=/usr/local/nixl @@ -543,7 +541,7 @@ RUN --mount=type=tmpfs,target=/tmp/lmcache-wheel \ c_ops="$(find /tmp/lmcache-wheel -type f -name 'c_ops*.so')"; \ test "$(find /tmp/lmcache-wheel -type f -name 'c_ops*.so' | wc -l)" -eq 1; \ expected="$(printf '%s' "${LMCACHE_ROCM_ARCH}" | tr ';' '\n' | sort -u)"; \ - actual="$(llvm-objdump --offloading "${c_ops}" \ + actual="$(/opt/rocm/llvm/bin/llvm-objdump --offloading "${c_ops}" \ | grep -oE 'gfx[0-9a-z]+' | sort -u)"; \ if [ "${actual}" != "${expected}" ]; then \ printf 'LMCache code objects mismatch\nexpected:\n%s\nactual:\n%s\n' \ @@ -551,10 +549,32 @@ RUN --mount=type=tmpfs,target=/tmp/lmcache-wheel \ exit 1; \ fi -# DeepEP build stage. ROCShmem is provided by TheRock SDK -# ROCSHMEM_DIR set to the SDK install path so DeepEP finds the headers, etc. -FROM base AS build_deepep -ENV ROCSHMEM_DIR=${ROCM_PATH} +# ROCShmem build stage - split from DeepEP so changing DEEPEP_BRANCH does not +# invalidate the slow ROCShmem build. +FROM base AS build_rocshmem +ARG ROCSHMEM_BRANCH="f0acb0c6" +ARG ROCSHMEM_REPO="https://github.com/ROCm/rocm-systems.git" +# DeepEP only supports gfx942 and gfx950; build ROCShmem for the same set so +# it can be linked against DeepEP without arch mismatches. +ARG DEEPEP_ROCM_ARCH="gfx942;gfx950" +ENV ROCM_PATH=/opt/rocm +ENV ROCSHMEM_DIR=/opt/rocshmem + +RUN --mount=type=cache,target=/root/.cache/ccache \ + git clone --no-checkout --filter=blob:none ${ROCSHMEM_REPO} \ + && cd rocm-systems \ + && git sparse-checkout set --cone projects/rocshmem \ + && git checkout ${ROCSHMEM_BRANCH} \ + && mkdir -p projects/rocshmem/build \ + && cd projects/rocshmem/build \ + && CC="ccache gcc" CXX="ccache g++" INSTALL_PREFIX=${ROCSHMEM_DIR} \ + bash ../scripts/build_configs/all_backends \ + -DROCM_PATH=${ROCM_PATH} \ + -DGPU_TARGETS="${DEEPEP_ROCM_ARCH}" \ + -DUSE_EXTERNAL_MPI=OFF + +# DeepEP build stage - depends on ROCShmem, builds the HIP kernel wheel. +FROM build_rocshmem AS build_deepep ARG DEEPEP_BRANCH="0f63d3e9" ARG DEEPEP_REPO="https://github.com/ROCm/DeepEP.git" ARG DEEPEP_NIC="cx7" @@ -571,8 +591,8 @@ RUN --mount=type=cache,target=/root/.cache/ccache \ && git checkout ${DEEPEP_BRANCH} \ && LDFLAGS="-fuse-ld=mold" MAX_JOBS="${MAX_JOBS:-$(nproc)}" python3 setup.py --variant rocm --rocm-explicit-ctx --nic ${DEEPEP_NIC} --rocm-gfx942-fp8-fnuz-max 224 bdist_wheel --dist-dir=/app/deep_install -# MoRI runtime dependencies live in Dockerfile.rocm so NIC backend changes do -# not force users to rebuild the long-lived Dockerfile.rocm_base image. +# MoRI runtime dependencies live in Dockerfile.rocm_72 so NIC backend changes do +# not force users to rebuild the long-lived Dockerfile.rocm_72_base image. FROM base AS mori_base ARG NIC_BACKEND ARG AINIC_VERSION @@ -673,9 +693,8 @@ RUN if [ "$GIT_REPO_CHECK" != "0" ]; then \ # Install the packaging helpers before modifying requirements/rocm.txt. RUN --mount=type=cache,target=/root/.cache/uv \ - { grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \ + grep -Fq "set(TRITON_KERNELS_TAG \"${ROCM_TRITON_KERNELS_COMMIT}\")" \ vllm/cmake/external_projects/triton_kernels.cmake \ - || echo "WARNING: Dockerfile.rock ROCM_TRITON_KERNELS_COMMIT (${ROCM_TRITON_KERNELS_COMMIT}) does not match cmake TRITON_KERNELS_TAG; the Docker build uses the Dockerfile pin, cmake governs only non-Docker builds"; } \ && cd vllm \ && uv pip install --system setuptools_scm regex @@ -731,12 +750,11 @@ COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/requirements /requir COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tests /tests COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/examples /examples -COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools/benchmark_helion_kernels.py /tools/benchmark_helion_kernels.py -COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools /tools COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile /docker/Dockerfile COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.cpu /docker/Dockerfile.cpu -COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/ -COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm_base /docker/Dockerfile.rocm_base +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm_72 /docker/ +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm_72_base /docker/ COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/ci-rocm.hcl /docker/ci-rocm.hcl COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/docker-bake.hcl /docker/docker-bake.hcl COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/docker-bake-rocm.hcl /docker/docker-bake-rocm.hcl @@ -777,6 +795,9 @@ RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \ --mount=type=bind,from=build_deepep,src=/app/deep_install,target=/deep_install \ uv pip install --system /nixl_install/*.whl /deep_install/*.whl +# Copy ROCShmem runtime libraries. +COPY --from=build_rocshmem /opt/rocshmem /opt/rocshmem + # RDMA userspace libraries plus FFmpeg dev libs needed by torchcodec. RUN timeout --kill-after=30s 5m apt-get update -q -y --error-on=any \ -o Acquire::Retries=3 \ @@ -809,8 +830,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ # Pre-install shared ROCm runtime dependencies. COPY requirements/common.txt requirements/rocm.txt /tmp/ci-base-requirements/ RUN --mount=type=cache,target=/root/.cache/uv \ - uv pip install --system -c /etc/rocm-constraints.txt \ - -r /tmp/ci-base-requirements/rocm.txt \ + uv pip install --system -r /tmp/ci-base-requirements/rocm.txt \ && rm -rf /tmp/ci-base-requirements # Enable fast and less brittle model downloads in tests. @@ -888,6 +908,8 @@ RUN mkdir src && mv vllm src/vllm # Catch GPU<->CPU syncs in execute_model/sample_tokens during tests. ENV VLLM_GPU_SYNC_CHECK=error +# Run the structural smoke test inside BuildKit, where the test image layers +# are already available. Export only a build-scoped success marker. FROM test AS test_smoke ARG ROCM_SMOKE_ID RUN --network=none \ @@ -934,37 +956,15 @@ RUN --mount=type=bind,from=export_vllm,src=/,target=/install \ --mount=type=cache,target=/root/.cache/uv \ cd /install \ && echo "opentelemetry-api<=1.40.0" > /tmp/otel-constraints.txt \ - && uv pip install --system -c /etc/rocm-constraints.txt -c /tmp/otel-constraints.txt -r requirements/rocm.txt \ + && uv pip install --system -c /tmp/otel-constraints.txt -r requirements/rocm.txt \ && pip uninstall -y vllm \ - && uv pip install --system -c /etc/rocm-constraints.txt -c /tmp/otel-constraints.txt *.whl \ + && uv pip install --system -c /tmp/otel-constraints.txt *.whl \ && rm /tmp/otel-constraints.txt # Install NIXL ROCm wheel RUN --mount=type=bind,from=build_nixl,src=/app/install,target=/nixl_install \ uv pip install --system /nixl_install/*.whl -# Recompute the base's triton fingerprint to detect resolver interference. -RUN EXPECTED=$(grep '^TRITON_SHA256: ' /app/versions.txt | cut -d' ' -f2) \ - && ACTUAL=$(find $(python3 -c 'import triton, os; print(os.path.dirname(triton.__file__))') \ - -type f \( -name '*.py' -o -name '*.so' \) | sort | xargs sha256sum | sha256sum | cut -d' ' -f1) \ - && echo "triton sha256 expected=${EXPECTED} actual=${ACTUAL}" \ - && test -n "${EXPECTED}" \ - && test "${EXPECTED}" = "${ACTUAL}" \ - && python3 -c "import torch; assert torch.version.hip, f'non-ROCm torch: {torch.__version__}'; print(f'torch={torch.__version__}')" - -# Recompute the base's pybind11 fingerprint to detect resolver interference. -RUN EXPECTED=$(grep '^PYBIND11_SHA256: ' /app/versions.txt | cut -d' ' -f2) \ - && EXPECTED_ABI=$(grep '^PYBIND11_INTERNALS_VERSION: ' /app/versions.txt | cut -d' ' -f2) \ - && INC=$(python3 -c 'import pybind11; print(pybind11.get_include())') \ - && ACTUAL=$(find "${INC}" -type f -name '*.h' | sort | xargs sha256sum | sha256sum | cut -d' ' -f1) \ - && ACTUAL_ABI=$(grep -rhE '^#[[:space:]]*define PYBIND11_INTERNALS_VERSION' "${INC}" | awk '{print $NF}') \ - && echo "pybind11 $(pip show pybind11 | awk '/^Version:/ {print $2}')" \ - "sha256 expected=${EXPECTED} actual=${ACTUAL}" \ - "internals expected=${EXPECTED_ABI} actual=${ACTUAL_ABI}" \ - && test -n "${EXPECTED}" \ - && test "${EXPECTED}" = "${ACTUAL}" \ - && test "${EXPECTED_ABI}" = "${ACTUAL_ABI}" - ARG COMMON_WORKDIR ARG BASE_IMAGE ARG NIC_BACKEND diff --git a/docker/Dockerfile.rocm_72_base b/docker/Dockerfile.rocm_72_base new file mode 100644 index 0000000000..b70434f090 --- /dev/null +++ b/docker/Dockerfile.rocm_72_base @@ -0,0 +1,479 @@ +ARG BASE_IMAGE=rocm/dev-ubuntu-22.04:7.2.3-complete +ARG TRITON_BRANCH="669b31a" # release/internal/3.8.x as of 09/17 +ARG TRITON_REPO="https://github.com/ROCm/triton.git" +ARG PYTORCH_BRANCH="733fca1" # release/2.13 as of 09/17 +ARG PYTORCH_REPO="https://github.com/ROCm/pytorch.git" +ARG PYTORCH_VISION_BRANCH="v0.28.0" +ARG PYTORCH_VISION_REPO="https://github.com/pytorch/vision.git" +ARG PYTORCH_AUDIO_BRANCH="v2.11.0" +ARG PYTORCH_AUDIO_REPO="https://github.com/pytorch/audio.git" +ARG FA_BRANCH="0e60e394" +ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git" +ARG AITER_BRANCH="v0.1.25" +ARG AITER_REPO="https://github.com/ROCm/aiter.git" +ARG MORI_BRANCH="v1.2.3.post1" +ARG MORI_REPO="https://github.com/ROCm/mori.git" +ARG ROCM_SYSTEMS_REPO="https://github.com/ROCm/rocm-systems.git" +ARG ROCPROFILER_SDK_COMMIT="2b22ab0195cc1461cd9abf3b969e9dd7c10af350" +ARG ROCM_RUNTIME_COMMIT="af70edc91f1dc5d228fd4c076df99e65e49c4521" + +# Sccache configuration (only used in release pipeline) +ARG USE_SCCACHE +ARG SCCACHE_DOWNLOAD_URL +ARG SCCACHE_ENDPOINT +ARG SCCACHE_BUCKET_NAME=vllm-build-sccache +ARG SCCACHE_REGION_NAME=us-west-2 +ARG SCCACHE_S3_NO_CREDENTIALS=0 + + +FROM ${BASE_IMAGE} AS base + +ENV PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV ROCM_PATH=/opt/rocm +ENV LD_LIBRARY_PATH=/opt/rocm/lib:/usr/local/lib: + +ARG PYTORCH_ROCM_ARCH=gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1200;gfx1201;gfx1150;gfx1151 +ENV PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH} +ENV AITER_ROCM_ARCH=gfx942;gfx950 +# Note: Do not set MORI_GPU_ARCHS here, it is automatically inferred at runtime + +# Required for RCCL in ROCm7.1 +ENV HSA_NO_SCRATCH_RECLAIM=1 + +ARG PYTHON_VERSION=3.12 +ENV PYTHON_VERSION=${PYTHON_VERSION} + +RUN mkdir -p /app +WORKDIR /app +ENV DEBIAN_FRONTEND=noninteractive + +# Install Python and other dependencies (plus gcc-13, selected later in build_pytorch). +RUN apt-get update -y \ + && apt-get install -y software-properties-common git curl sudo vim less libgfortran5 libopenmpi-dev libpci-dev liblzma-dev pkg-config \ + && for i in 1 2 3; do \ + add-apt-repository -y ppa:deadsnakes/ppa && add-apt-repository -y ppa:ubuntu-toolchain-r/test && break || \ + { echo "Attempt $i failed, retrying in 5s..."; sleep 5; }; \ + done \ + && apt-get update -y \ + && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv \ + python${PYTHON_VERSION}-lib2to3 python-is-python3 gcc-13 g++-13 \ + && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \ + && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \ + && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \ + && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \ + && python3 --version && python3 -m pip --version + +RUN pip install -U packaging 'cmake<4' ninja wheel 'setuptools<80' 'pybind11<3.1.0' Cython +RUN apt-get update && apt-get install -y libjpeg-dev libsox-dev libsox-fmt-all sox && rm -rf /var/lib/apt/lists/* + +# Install sccache if USE_SCCACHE is enabled (for release builds) +ARG USE_SCCACHE +ARG SCCACHE_DOWNLOAD_URL +ARG SCCACHE_ENDPOINT +ARG SCCACHE_BUCKET_NAME +ARG SCCACHE_REGION_NAME +ARG SCCACHE_S3_NO_CREDENTIALS +RUN if [ "$USE_SCCACHE" = "1" ]; then \ + echo "Installing sccache..." \ + && SCCACHE_ARCH="x86_64" \ + && SCCACHE_VERSION="v0.8.1" \ + && SCCACHE_DL_URL="${SCCACHE_DOWNLOAD_URL:-https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}-unknown-linux-musl.tar.gz}" \ + && curl -L -o /tmp/sccache.tar.gz ${SCCACHE_DL_URL} \ + && tar -xzf /tmp/sccache.tar.gz -C /tmp \ + && mv /tmp/sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}-unknown-linux-musl/sccache /usr/bin/sccache \ + && chmod +x /usr/bin/sccache \ + && rm -rf /tmp/sccache.tar.gz /tmp/sccache-${SCCACHE_VERSION}-${SCCACHE_ARCH}-unknown-linux-musl \ + && sccache --version; \ + fi + +# Setup sccache for HIP compilation via HIP_CLANG_PATH +# This creates wrapper scripts in a separate directory and points HIP to use them +# This avoids modifying the original ROCm binaries which can break detection +# NOTE: HIP_CLANG_PATH is NOT set as ENV to avoid affecting downstream images (Dockerfile.rocm_72) +# Instead, each build stage should export HIP_CLANG_PATH=/opt/sccache-wrappers if USE_SCCACHE=1 +RUN if [ "$USE_SCCACHE" = "1" ]; then \ + echo "Setting up sccache wrappers for HIP compilation..." \ + && mkdir -p /opt/sccache-wrappers \ + && printf '#!/bin/bash\nexec sccache /opt/rocm/lib/llvm/bin/clang++ "$@"\n' > /opt/sccache-wrappers/clang++ \ + && chmod +x /opt/sccache-wrappers/clang++ \ + && printf '#!/bin/bash\nexec sccache /opt/rocm/lib/llvm/bin/clang "$@"\n' > /opt/sccache-wrappers/clang \ + && chmod +x /opt/sccache-wrappers/clang \ + && echo "sccache wrappers created in /opt/sccache-wrappers"; \ + fi + +# Set sccache environment variables only when USE_SCCACHE=1 +# This prevents S3 config from leaking into images when sccache is not used +ARG USE_SCCACHE +ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET_NAME}} +ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}} +ENV SCCACHE_S3_NO_CREDENTIALS=${USE_SCCACHE:+${SCCACHE_S3_NO_CREDENTIALS}} +ENV SCCACHE_IDLE_TIMEOUT=${USE_SCCACHE:+0} + + +### +### Triton Build +### +FROM base AS build_triton +ARG TRITON_BRANCH +ARG TRITON_REPO +RUN git clone ${TRITON_REPO} +RUN cd triton \ + && git checkout ${TRITON_BRANCH} \ + && git config --global user.email "you@example.com" && git config --global user.name "Your Name" \ + && if [ ! -f setup.py ]; then cd python; fi \ + && MAX_JOBS=16 python3 setup.py bdist_wheel --dist-dir=dist \ + && mkdir -p /app/install && cp dist/*.whl /app/install + + +### +### AMD SMI Build +### +FROM base AS build_amdsmi +RUN cd /opt/rocm/share/amd_smi \ + && pip wheel . --wheel-dir=dist +RUN mkdir -p /app/install && cp /opt/rocm/share/amd_smi/dist/*.whl /app/install + + +### +### Pytorch build +### +FROM base AS build_pytorch +ARG PYTORCH_BRANCH +ARG PYTORCH_REPO +ARG USE_SCCACHE + +RUN apt-get update && apt-get install -y pkg-config liblzma-dev +# torch 2.12 emits float->__bf16 libcalls (__truncsfbf2) absent from gcc-11 libgcc; build with gcc-13. +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 --slave /usr/bin/g++ g++ /usr/bin/g++-13 +RUN git clone ${PYTORCH_REPO} pytorch +RUN cd pytorch && git checkout ${PYTORCH_BRANCH} +RUN cd pytorch \ + && pip install -r requirements.txt && git submodule update --init --recursive +RUN cd pytorch && python3 tools/amd_build/build_amd.py \ + && if [ "$USE_SCCACHE" = "1" ]; then \ + export HIP_CLANG_PATH=/opt/sccache-wrappers \ + && export CMAKE_C_COMPILER_LAUNCHER=sccache \ + && export CMAKE_CXX_COMPILER_LAUNCHER=sccache \ + && sccache --show-stats; \ + fi \ + && CMAKE_PREFIX_PATH=$(python3 -c 'import sys; print(sys.prefix)') python3 setup.py bdist_wheel --dist-dir=dist \ + && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi +RUN mkdir -p /app/install && cp /app/pytorch/dist/*.whl /app/install + + +### +### Shared PyTorch runtime for dependent builds +### +FROM base AS build_pytorch_runtime +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 --slave /usr/bin/g++ g++ /usr/bin/g++-13 +RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install,ro \ + pip install /install/*.whl + + +### +### TorchVision build +### +FROM build_pytorch_runtime AS build_torchvision +ARG PYTORCH_VISION_BRANCH +ARG PYTORCH_VISION_REPO +ARG USE_SCCACHE + +RUN git clone ${PYTORCH_VISION_REPO} vision +RUN cd vision && git checkout ${PYTORCH_VISION_BRANCH} \ + && if [ "$USE_SCCACHE" = "1" ]; then \ + export HIP_CLANG_PATH=/opt/sccache-wrappers \ + && export CMAKE_C_COMPILER_LAUNCHER=sccache \ + && export CMAKE_CXX_COMPILER_LAUNCHER=sccache; \ + fi \ + && python3 setup.py bdist_wheel --dist-dir=dist \ + && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi +RUN mkdir -p /app/install && cp /app/vision/dist/*.whl /app/install + + +### +### TorchAudio build +### +FROM build_pytorch_runtime AS build_torchaudio +ARG PYTORCH_AUDIO_BRANCH +ARG PYTORCH_AUDIO_REPO +ARG USE_SCCACHE + +RUN git clone ${PYTORCH_AUDIO_REPO} audio +RUN cd audio && git checkout ${PYTORCH_AUDIO_BRANCH} \ + && git submodule update --init --recursive +RUN cd audio && pip install -r requirements.txt +RUN cd audio \ + && if [ "$USE_SCCACHE" = "1" ]; then \ + export HIP_CLANG_PATH=/opt/sccache-wrappers \ + && export CMAKE_C_COMPILER_LAUNCHER=sccache \ + && export CMAKE_CXX_COMPILER_LAUNCHER=sccache; \ + fi \ + && python3 setup.py bdist_wheel --dist-dir=dist \ + && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi +RUN mkdir -p /app/install && cp /app/audio/dist/*.whl /app/install + + +### +### MORI Build +### +FROM base AS build_mori +ARG MORI_BRANCH +ARG MORI_REPO +RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \ + pip install /install/*.whl +RUN git clone ${MORI_REPO} +RUN cd mori \ + && git checkout ${MORI_BRANCH} \ + && git submodule update --init --recursive \ + && python3 setup.py bdist_wheel --dist-dir=dist && ls /app/mori/dist/*.whl +RUN mkdir -p /app/install && cp /app/mori/dist/*.whl /app/install + + +### +### FlashAttention Build +### +FROM base AS build_fa +ARG FA_BRANCH +ARG FA_REPO +ARG USE_SCCACHE +RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \ + pip install /install/*.whl +RUN git clone ${FA_REPO} +RUN cd flash-attention \ + && git checkout ${FA_BRANCH} \ + && git submodule update --init \ + && if [ "$USE_SCCACHE" = "1" ]; then \ + export HIP_CLANG_PATH=/opt/sccache-wrappers \ + && sccache --show-stats; \ + fi \ + && GPU_ARCHS=$(echo ${PYTORCH_ROCM_ARCH} | sed -e 's/;gfx1[0-9]\{3\}//g') python3 setup.py bdist_wheel --dist-dir=dist \ + && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi +RUN mkdir -p /app/install && cp /app/flash-attention/dist/*.whl /app/install + + +### +### AITER Build +### +FROM base AS build_aiter +ARG AITER_BRANCH +ARG AITER_REPO +ARG USE_SCCACHE +RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \ + pip install /install/*.whl +RUN git clone --recursive --branch ${AITER_BRANCH} ${AITER_REPO} +RUN cd aiter \ + && git submodule update --init --recursive \ + && pip install -r requirements.txt +RUN pip install pyyaml && cd aiter \ + && if [ "$USE_SCCACHE" = "1" ]; then \ + export HIP_CLANG_PATH=/opt/sccache-wrappers \ + && sccache --show-stats; \ + fi \ + && PREBUILD_KERNELS=1 AITER_USE_SYSTEM_TRITON=1 GPU_ARCHS=${AITER_ROCM_ARCH} python3 setup.py bdist_wheel --dist-dir=dist \ + && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi \ + && ls /app/aiter/dist/*.whl +RUN mkdir -p /app/install && cp /app/aiter/dist/*.whl /app/install + + +### +### Final Build +### + +# Wheel release stage - +# only includes dependencies used by wheel release pipeline +FROM base AS debs_wheel_release +RUN mkdir /app/debs +RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_amdsmi,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_torchvision,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_torchaudio,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs + +# Full debs stage - includes Mori (used by Docker releases) +FROM base AS debs +RUN mkdir /app/debs +RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_amdsmi,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_torchvision,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_torchaudio,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_mori,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs + +### +### ROCr + CLR build +### +# Pull in updates to ROCr + clr from ROCm 7.2.4 and beyond. +FROM base AS build_rocm_runtime +ARG ROCM_SYSTEMS_REPO +ARG ROCM_RUNTIME_COMMIT +RUN apt-get update \ + && apt-get install -y --no-install-recommends g++ \ + libelf-dev libdrm-dev libnuma-dev libdw-dev xxd \ + && rm -rf /var/lib/apt/lists/* +RUN pip install --no-cache-dir CppHeaderParser +RUN git init -q /src && cd /src \ + && git remote add origin ${ROCM_SYSTEMS_REPO} \ + && git config core.sparseCheckout true \ + && git config remote.origin.promisor true \ + && git config remote.origin.partialclonefilter blob:none \ + && git sparse-checkout init --cone \ + && git sparse-checkout set projects/rocr-runtime projects/clr projects/hip shared cmake \ + && git fetch --filter=blob:none --depth 1 origin ${ROCM_RUNTIME_COMMIT} \ + && git checkout -q FETCH_HEAD +# ROCr's trap_handler and blit_shaders call find_package(Clang/LLVM REQUIRED) only to get +# IMPORTED executables. The ROCm dev image ships the binaries but no CMake package config, so +# supply one pointing at them. +RUN mkdir -p /opt/rocm-cmake-shim \ + && printf 'if(NOT TARGET clang)\n add_executable(clang IMPORTED GLOBAL)\n set_target_properties(clang PROPERTIES IMPORTED_LOCATION "/opt/rocm/llvm/bin/clang")\nendif()\nset(Clang_PACKAGE_VERSION "rocm-image-shim")\n' > /opt/rocm-cmake-shim/ClangConfig.cmake \ + && printf 'if(NOT TARGET llvm-objcopy)\n add_executable(llvm-objcopy IMPORTED GLOBAL)\n set_target_properties(llvm-objcopy PROPERTIES IMPORTED_LOCATION "/opt/rocm/llvm/bin/llvm-objcopy")\nendif()\nset(LLVM_FOUND TRUE)\nset(LLVM_PACKAGE_VERSION "rocm-image-shim")\n' > /opt/rocm-cmake-shim/LLVMConfig.cmake +# ROCr first, and installed into /opt/rocm, because the clr build below needs its headers +RUN cd /src/projects/rocr-runtime \ + && cmake -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH=/opt/rocm \ + -DCMAKE_INSTALL_PREFIX=/opt/rocm \ + -DClang_DIR=/opt/rocm-cmake-shim \ + -DLLVM_DIR=/opt/rocm-cmake-shim \ + && cmake --build build --parallel 16 \ + && cmake --install build \ + && cmake --install build --prefix /rocr-install --strip +RUN cd /src/projects/clr \ + && cmake -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCLR_BUILD_HIP=ON \ + -DCLR_BUILD_OCL=OFF \ + -DHIP_COMMON_DIR=/src/projects/hip \ + -DROCM_PATH=/opt/rocm \ + -DCMAKE_PREFIX_PATH=/opt/rocm \ + -DCMAKE_INSTALL_PREFIX=/opt/rocm \ + -DLLVM_DIR=/opt/rocm-cmake-shim \ + -DHIP_LLVM_ROOT=/opt/rocm/llvm \ + && cmake --build build --parallel 16 \ + && cmake --install build --prefix /clr-install --strip +RUN mkdir -p /staging/lib \ + && cp -P /rocr-install/lib/libhsa-runtime64.so* /staging/lib/ \ + && cp -P /clr-install/lib/libamdhip64.so* /staging/lib/ \ + && ls -l /staging/lib + +# rocprofiler-sdk 1.3.2 fixes torch 2.12 profiler on ROCm 7.2.3 +FROM base AS build_rocprofiler_sdk +ARG ROCM_SYSTEMS_REPO +ARG ROCPROFILER_SDK_COMMIT +ARG USE_SCCACHE +RUN apt-get update \ + && apt-get install -y --no-install-recommends libdw-dev libelf-dev libsqlite3-dev libzstd-dev \ + && rm -rf /var/lib/apt/lists/* +RUN git init -q /src && cd /src \ + && git remote add origin ${ROCM_SYSTEMS_REPO} \ + && git config core.sparseCheckout true \ + && git config remote.origin.promisor true \ + && git config remote.origin.partialclonefilter blob:none \ + && git sparse-checkout init --cone \ + && git sparse-checkout set projects/rocprofiler-sdk shared cmake \ + && git fetch --filter=blob:none --depth 1 origin ${ROCPROFILER_SDK_COMMIT} \ + && git checkout -q FETCH_HEAD +# Patch the ROCm 7.2.3 build of rocprofiler-sdk to include queue-interposition fixes that are +# absent from the pinned 7.14 build. This resolves hangs on torch.profiler on ROCm. +# TODO: Drop these once ${ROCPROFILER_SDK_COMMIT} is bumped to a build that already +# contains them, such as ROCm 10.0. See https://github.com/vllm-project/vllm/issues/54087. +ARG ROCPROFILER_SDK_PR_7796=fae7fd57ffb8a76e811d6410b8a0cee1e94d710a +ARG ROCPROFILER_SDK_PR_7924=f8db22adbe17686a9b05ae03367dd1ec73011653 +RUN cd /src \ + && HSA_PATHS='projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/*' \ + && curl -fsSL -o /tmp/qi-7796.patch "${ROCM_SYSTEMS_REPO%.git}/commit/${ROCPROFILER_SDK_PR_7796}.patch" \ + && curl -fsSL -o /tmp/qi-7924.patch "${ROCM_SYSTEMS_REPO%.git}/commit/${ROCPROFILER_SDK_PR_7924}.patch" \ + && git apply -p1 --include="$HSA_PATHS" --verbose /tmp/qi-7796.patch \ + && git apply -p1 --include="$HSA_PATHS" --verbose /tmp/qi-7924.patch +RUN cd /src/projects/rocprofiler-sdk \ + && if [ "$USE_SCCACHE" = "1" ]; then \ + export CMAKE_C_COMPILER_LAUNCHER=sccache \ + && export CMAKE_CXX_COMPILER_LAUNCHER=sccache; \ + fi \ + && cmake -B build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_PREFIX_PATH=/opt/rocm \ + -DCMAKE_INSTALL_PREFIX=/opt/rocm \ + -DROCPROFILER_BUILD_TESTS=OFF \ + -DROCPROFILER_BUILD_SAMPLES=OFF \ + -DROCPROFILER_BUILD_DOCS=OFF \ + && cmake --build build --parallel "$(nproc)" \ + && cmake --install build --prefix /staging --strip + +FROM base AS final +RUN --mount=type=bind,from=debs,src=/app/debs,target=/install \ + pip install /install/*.whl + +COPY --from=build_rocprofiler_sdk /staging/ /opt/rocm/ +ENV ROCPROFILER_QUEUE_INTERPOSITION=0 + +COPY --from=build_rocm_runtime /staging/ /opt/rocm/ +# Hard link the patched library onto every stock file name so that both the +# SONAME lookup and the hard coded paths resolve to it. +RUN set -eux; \ + cd /opt/rocm/lib; \ + ours_hsa="$(readlink -f libhsa-runtime64.so.1)"; \ + ours_hip="$(readlink -f libamdhip64.so.7)"; \ + for f in libhsa-runtime64.so.1.*; do \ + [ "$f" = "${ours_hsa##*/}" ] || ln -f "$ours_hsa" "$f"; \ + done; \ + for f in libamdhip64.so.7.*; do \ + [ "$f" = "${ours_hip##*/}" ] || ln -f "$ours_hip" "$f"; \ + done + +RUN ldconfig + +ARG BASE_IMAGE +ARG TRITON_BRANCH +ARG TRITON_REPO +ARG PYTORCH_BRANCH +ARG PYTORCH_VISION_BRANCH +ARG PYTORCH_REPO +ARG PYTORCH_VISION_REPO +ARG PYTORCH_AUDIO_BRANCH +ARG PYTORCH_AUDIO_REPO +ARG FA_BRANCH +ARG FA_REPO +ARG AITER_BRANCH +ARG AITER_REPO +ARG MORI_BRANCH +ARG MORI_REPO +ARG ROCM_SYSTEMS_REPO +ARG ROCPROFILER_SDK_COMMIT +ARG ROCM_RUNTIME_COMMIT +RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \ + && echo "ROCM_SYSTEMS_REPO: ${ROCM_SYSTEMS_REPO}" >> /app/versions.txt \ + && echo "ROCPROFILER_SDK_COMMIT: ${ROCPROFILER_SDK_COMMIT}" >> /app/versions.txt \ + && echo "ROCM_RUNTIME_COMMIT: ${ROCM_RUNTIME_COMMIT}" >> /app/versions.txt \ + && echo "TRITON_BRANCH: ${TRITON_BRANCH}" >> /app/versions.txt \ + && echo "TRITON_REPO: ${TRITON_REPO}" >> /app/versions.txt \ + && echo "PYTORCH_BRANCH: ${PYTORCH_BRANCH}" >> /app/versions.txt \ + && echo "PYTORCH_VISION_BRANCH: ${PYTORCH_VISION_BRANCH}" >> /app/versions.txt \ + && echo "PYTORCH_REPO: ${PYTORCH_REPO}" >> /app/versions.txt \ + && echo "PYTORCH_VISION_REPO: ${PYTORCH_VISION_REPO}" >> /app/versions.txt \ + && echo "PYTORCH_AUDIO_BRANCH: ${PYTORCH_AUDIO_BRANCH}" >> /app/versions.txt \ + && echo "PYTORCH_AUDIO_REPO: ${PYTORCH_AUDIO_REPO}" >> /app/versions.txt \ + && echo "FA_BRANCH: ${FA_BRANCH}" >> /app/versions.txt \ + && echo "FA_REPO: ${FA_REPO}" >> /app/versions.txt \ + && echo "AITER_BRANCH: ${AITER_BRANCH}" >> /app/versions.txt \ + && echo "AITER_REPO: ${AITER_REPO}" >> /app/versions.txt \ + && echo "MORI_BRANCH: ${MORI_BRANCH}" >> /app/versions.txt \ + && echo "MORI_REPO: ${MORI_REPO}" >> /app/versions.txt diff --git a/docker/Dockerfile.rocm_base b/docker/Dockerfile.rocm_base index 33dd0f6b10..933f112e8e 100644 --- a/docker/Dockerfile.rocm_base +++ b/docker/Dockerfile.rocm_base @@ -1,12 +1,11 @@ -ARG BASE_IMAGE=rocm/dev-ubuntu-22.04:7.2.3-complete -ARG TRITON_BRANCH="669b31a" # release/internal/3.8.x as of 09/17 +ARG BASE_IMAGE=ubuntu:22.04 +ARG PYTHON_VERSION=3.12 +ARG TORCH_VERSION=2.12.0+rocm10.0.0 +ARG TORCHVISION_VERSION=0.27.0+rocm10.0.0 +ARG TORCHAUDIO_VERSION=2.11.0+rocm10.0.0 +ARG TRITON_BRANCH="669b31ac" # release/internal/3.8.x as of 09/29/2026 +ARG TRITON_VERSION=3.8.0+git669b31ac ARG TRITON_REPO="https://github.com/ROCm/triton.git" -ARG PYTORCH_BRANCH="733fca1" # release/2.13 as of 09/17 -ARG PYTORCH_REPO="https://github.com/ROCm/pytorch.git" -ARG PYTORCH_VISION_BRANCH="v0.28.0" -ARG PYTORCH_VISION_REPO="https://github.com/pytorch/vision.git" -ARG PYTORCH_AUDIO_BRANCH="v2.11.0" -ARG PYTORCH_AUDIO_REPO="https://github.com/pytorch/audio.git" ARG FA_BRANCH="0e60e394" ARG FA_REPO="https://github.com/Dao-AILab/flash-attention.git" ARG AITER_BRANCH="v0.1.25" @@ -14,8 +13,18 @@ ARG AITER_REPO="https://github.com/ROCm/aiter.git" ARG MORI_BRANCH="v1.2.3.post1" ARG MORI_REPO="https://github.com/ROCm/mori.git" ARG ROCM_SYSTEMS_REPO="https://github.com/ROCm/rocm-systems.git" -ARG ROCPROFILER_SDK_COMMIT="2b22ab0195cc1461cd9abf3b969e9dd7c10af350" -ARG ROCM_RUNTIME_COMMIT="af70edc91f1dc5d228fd4c076df99e65e49c4521" +ARG ROCM_RUNTIME_COMMIT="25e14349f2606a6b95c4ccd45eb92a1835646cf2" + + +# TheRock configuration +ARG SITE_PACKAGES=/usr/local/lib/python${PYTHON_VERSION}/dist-packages +ARG ROCM_RELEASE_WHEELS_MULTIARCH_URL="https://stable.repo.amd.com/rocm/whl-next/" # For pre-built GPU images, packages, etc. +ARG ROCM_NIGHTLY_WHEELS_MULTIARCH_URL="https://nightly.repo.amd.com/rocm/whl-next/" # For torch, triton, etc. wheels with latest fixes. +ARG ROCM_SDK_VERSION=10.0.0 + +# Build configuration +ARG PYTORCH_ROCM_ARCH="gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1200;gfx1201;gfx1150;gfx1151" +ARG AITER_ROCM_ARCH="gfx942;gfx950" # Sccache configuration (only used in release pipeline) ARG USE_SCCACHE @@ -25,47 +34,128 @@ ARG SCCACHE_BUCKET_NAME=vllm-build-sccache ARG SCCACHE_REGION_NAME=us-west-2 ARG SCCACHE_S3_NO_CREDENTIALS=0 - FROM ${BASE_IMAGE} AS base -ENV PATH=/opt/rocm/llvm/bin:/opt/rocm/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -ENV ROCM_PATH=/opt/rocm -ENV LD_LIBRARY_PATH=/opt/rocm/lib:/usr/local/lib: +ARG PYTORCH_ROCM_ARCH +ENV PYTORCH_ROCM_ARCH="${PYTORCH_ROCM_ARCH}" +ARG AITER_ROCM_ARCH +ENV AITER_ROCM_ARCH="${AITER_ROCM_ARCH}" -ARG PYTORCH_ROCM_ARCH=gfx90a;gfx942;gfx950;gfx1100;gfx1101;gfx1200;gfx1201;gfx1150;gfx1151 -ENV PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH} -ENV AITER_ROCM_ARCH=gfx942;gfx950 -# Note: Do not set MORI_GPU_ARCHS here, it is automatically inferred at runtime +ARG SITE_PACKAGES +ENV SITE_PACKAGES=${SITE_PACKAGES} +ENV SDK_DEV="$SITE_PACKAGES/_rocm_sdk_devel" \ + SDK_CORE="$SITE_PACKAGES/_rocm_sdk_core" +ENV ROCM_PATH="$SDK_DEV" \ + ROCM_HOME="$SDK_DEV" \ + HIP_PATH="$SDK_DEV" \ + HIP_CLANG_PATH="$SDK_DEV/lib/llvm/bin" \ + HIP_DEVICE_LIB_PATH="$SDK_CORE/lib/llvm/amdgcn/bitcode" +ENV LD_LIBRARY_PATH="$SDK_DEV/lib:$SDK_DEV/lib/rocm_sysdeps/lib:$SDK_CORE/lib:$SDK_CORE/lib/rocm_sysdeps/lib" +ENV PATH="$SDK_DEV/bin:$SDK_DEV/lib/llvm/bin:$SDK_CORE/bin:$PATH" # Required for RCCL in ROCm7.1 ENV HSA_NO_SCRATCH_RECLAIM=1 -ARG PYTHON_VERSION=3.12 +ARG PYTHON_VERSION ENV PYTHON_VERSION=${PYTHON_VERSION} RUN mkdir -p /app WORKDIR /app ENV DEBIAN_FRONTEND=noninteractive -# Install Python and other dependencies (plus gcc-13, selected later in build_pytorch). +# Install Python and other dependencies RUN apt-get update -y \ - && apt-get install -y software-properties-common git curl sudo vim less libgfortran5 libopenmpi-dev libpci-dev liblzma-dev pkg-config \ + && apt-get install -y software-properties-common build-essential git curl sudo vim less libgfortran5 libopenmpi-dev libpci-dev liblzma-dev pkg-config libdrm-dev libnuma-dev kmod \ && for i in 1 2 3; do \ - add-apt-repository -y ppa:deadsnakes/ppa && add-apt-repository -y ppa:ubuntu-toolchain-r/test && break || \ + add-apt-repository -y ppa:deadsnakes/ppa && break || \ { echo "Attempt $i failed, retrying in 5s..."; sleep 5; }; \ done \ && apt-get update -y \ && apt-get install -y python${PYTHON_VERSION} python${PYTHON_VERSION}-dev python${PYTHON_VERSION}-venv \ - python${PYTHON_VERSION}-lib2to3 python-is-python3 gcc-13 g++-13 \ + python${PYTHON_VERSION}-lib2to3 python-is-python3 \ && update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VERSION} 1 \ && update-alternatives --set python3 /usr/bin/python${PYTHON_VERSION} \ && ln -sf /usr/bin/python${PYTHON_VERSION}-config /usr/bin/python3-config \ && curl -sS https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION} \ && python3 --version && python3 -m pip --version -RUN pip install -U packaging 'cmake<4' ninja wheel 'setuptools<80' 'pybind11<3.1.0' Cython +RUN pip install -U packaging 'cmake<4' ninja wheel 'setuptools<80' pybind11==3.0.4 Cython RUN apt-get update && apt-get install -y libjpeg-dev libsox-dev libsox-fmt-all sox && rm -rf /var/lib/apt/lists/* +# Install ROCm SDK via TheRock +ARG ROCM_RELEASE_WHEELS_MULTIARCH_URL +ARG ROCM_NIGHTLY_WHEELS_MULTIARCH_URL +ARG ROCM_SDK_VERSION +RUN pip install --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \ + --extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \ + "rocm[libraries,devel]${ROCM_SDK_VERSION:+==${ROCM_SDK_VERSION}}" \ + && rocm-sdk init + +# Install PyTorch wheels (nightly for latest torch, release as fallback) +ARG TORCH_VERSION +ARG TORCHAUDIO_VERSION +ARG TORCHVISION_VERSION +RUN pip install \ + --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \ + --extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \ + "torch==${TORCH_VERSION}" \ + "torchaudio==${TORCHAUDIO_VERSION}" \ + "torchvision==${TORCHVISION_VERSION}" + +# Install the AMD-compiled device kernels corresponding to PYTORCH_ROCM_ARCH: + +# Possible values that can be in PYTORCH_ROCM_ARCH: +# +# gfx90a (MI210/250), gfx942 (MI300), gfx950 (MI350), gfx1100 (7900XTX), +# gfx1101 (7800XT), gfx1200 (9070XT), gfx1201 (9070), gfx1150/1151 (Strix APU) + +# For each arch in PYTORCH_ROCM_ARCH, install the precompiled GPU code that +# PyTorch needs to run on that specific chip. +SHELL ["/bin/bash", "-c"] +RUN TORCH_VER=$(python3 -c "import torch; print(torch.__version__)") \ + && for arch in $(echo "${PYTORCH_ROCM_ARCH}" | tr ';' ' '); do \ + echo "Installing device packages for ${arch}" \ + && pip install --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \ + --extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \ + "torch[device-${arch}]==${TORCH_VER}" \ + "torchvision[device-${arch}]==${TORCHVISION_VERSION}" \ + "rocm[device-${arch}]==${ROCM_SDK_VERSION}"; \ + done + +# Remove gfx90a xnack kpacks (ASAN artifacts that cause hipErrorInvalidImage on MI250) +RUN rm -f "${SITE_PACKAGES}/torch/.kpack/torch_gfx90a:xnack+.kpack" \ + "${SITE_PACKAGES}/torch/.kpack/torch_gfx90a:xnack-.kpack" + +RUN cd $SITE_PACKAGES/_rocm_sdk_core/share/amd_smi && pip install . +ENV PYTHONPATH=$SITE_PACKAGES/_rocm_sdk_core/share/amd_smi + +# TheRock runtime fixes +RUN if [ -f "${SITE_PACKAGES}/rocm_sdk/__init__.py" ]; then \ + sed -i 's/rtld_global: bool = True/rtld_global: bool = False/g' "${SITE_PACKAGES}/rocm_sdk/__init__.py"; \ + fi \ + && if [ -f "${SITE_PACKAGES}/_rocm_sdk_core/lib/librocm_smi64.so.1" ] \ + && [ ! -e "${ROCM_PATH}/lib/librocm_smi64.so.7" ]; then \ + ln -sfn "${SITE_PACKAGES}/_rocm_sdk_core/lib/librocm_smi64.so.1" "${ROCM_PATH}/lib/librocm_smi64.so.7"; \ + fi \ + && printf '%s\n%s\n' "${ROCM_PATH}/lib" "${SITE_PACKAGES}/_rocm_sdk_core/lib" > /etc/ld.so.conf.d/rocm-pip.conf \ + && for f in ${SITE_PACKAGES}/_rocm_sdk_core/lib/libamdhip64.so*; do \ + bn=$(basename "$f"); \ + if [ -f "${ROCM_PATH}/lib/${bn}" ] && [ -f "$f" ] && [ ! -L "$f" ]; then \ + rm -f "$f"; \ + ln -sfn "${ROCM_PATH}/lib/${bn}" "$f"; \ + fi; \ + done \ + && for f in librocprofiler-sdk.so.1 librocprofiler-register.so.0 \ + librocprofiler-sdk-roctx.so.1; do \ + if [ -e "${SITE_PACKAGES}/_rocm_sdk_core/lib/${f}" ] \ + && [ -e "${SITE_PACKAGES}/_rocm_sdk_devel/lib/${f}" ]; then \ + rm -f "${SITE_PACKAGES}/_rocm_sdk_core/lib/${f}"; \ + ln -sfn "${SITE_PACKAGES}/_rocm_sdk_devel/lib/${f}" \ + "${SITE_PACKAGES}/_rocm_sdk_core/lib/${f}"; \ + fi; \ + done \ + && ldconfig + # Install sccache if USE_SCCACHE is enabled (for release builds) ARG USE_SCCACHE ARG SCCACHE_DOWNLOAD_URL @@ -94,141 +184,35 @@ RUN if [ "$USE_SCCACHE" = "1" ]; then \ RUN if [ "$USE_SCCACHE" = "1" ]; then \ echo "Setting up sccache wrappers for HIP compilation..." \ && mkdir -p /opt/sccache-wrappers \ - && printf '#!/bin/bash\nexec sccache /opt/rocm/lib/llvm/bin/clang++ "$@"\n' > /opt/sccache-wrappers/clang++ \ + && printf '#!/bin/bash\nexec sccache %s/lib/llvm/bin/clang++ "$@"\n' "${ROCM_PATH}" > /opt/sccache-wrappers/clang++ \ && chmod +x /opt/sccache-wrappers/clang++ \ - && printf '#!/bin/bash\nexec sccache /opt/rocm/lib/llvm/bin/clang "$@"\n' > /opt/sccache-wrappers/clang \ + && printf '#!/bin/bash\nexec sccache %s/lib/llvm/bin/clang "$@"\n' "${ROCM_PATH}" > /opt/sccache-wrappers/clang \ && chmod +x /opt/sccache-wrappers/clang \ && echo "sccache wrappers created in /opt/sccache-wrappers"; \ fi -# Set sccache environment variables only when USE_SCCACHE=1 -# This prevents S3 config from leaking into images when sccache is not used -ARG USE_SCCACHE ENV SCCACHE_BUCKET=${USE_SCCACHE:+${SCCACHE_BUCKET_NAME}} ENV SCCACHE_REGION=${USE_SCCACHE:+${SCCACHE_REGION_NAME}} ENV SCCACHE_S3_NO_CREDENTIALS=${USE_SCCACHE:+${SCCACHE_S3_NO_CREDENTIALS}} ENV SCCACHE_IDLE_TIMEOUT=${USE_SCCACHE:+0} -### -### Triton Build -### -FROM base AS build_triton -ARG TRITON_BRANCH -ARG TRITON_REPO -RUN git clone ${TRITON_REPO} -RUN cd triton \ - && git checkout ${TRITON_BRANCH} \ - && git config --global user.email "you@example.com" && git config --global user.name "Your Name" \ - && if [ ! -f setup.py ]; then cd python; fi \ - && MAX_JOBS=16 python3 setup.py bdist_wheel --dist-dir=dist \ - && mkdir -p /app/install && cp dist/*.whl /app/install - - -### -### AMD SMI Build -### -FROM base AS build_amdsmi -RUN cd /opt/rocm/share/amd_smi \ - && pip wheel . --wheel-dir=dist -RUN mkdir -p /app/install && cp /opt/rocm/share/amd_smi/dist/*.whl /app/install - - -### -### Pytorch build -### -FROM base AS build_pytorch -ARG PYTORCH_BRANCH -ARG PYTORCH_REPO -ARG USE_SCCACHE - -RUN apt-get update && apt-get install -y pkg-config liblzma-dev -# torch 2.12 emits float->__bf16 libcalls (__truncsfbf2) absent from gcc-11 libgcc; build with gcc-13. -RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 --slave /usr/bin/g++ g++ /usr/bin/g++-13 -RUN git clone ${PYTORCH_REPO} pytorch -RUN cd pytorch && git checkout ${PYTORCH_BRANCH} -RUN cd pytorch \ - && pip install -r requirements.txt && git submodule update --init --recursive -RUN cd pytorch && python3 tools/amd_build/build_amd.py \ - && if [ "$USE_SCCACHE" = "1" ]; then \ - export HIP_CLANG_PATH=/opt/sccache-wrappers \ - && export CMAKE_C_COMPILER_LAUNCHER=sccache \ - && export CMAKE_CXX_COMPILER_LAUNCHER=sccache \ - && sccache --show-stats; \ - fi \ - && CMAKE_PREFIX_PATH=$(python3 -c 'import sys; print(sys.prefix)') python3 setup.py bdist_wheel --dist-dir=dist \ - && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi -RUN mkdir -p /app/install && cp /app/pytorch/dist/*.whl /app/install - - -### -### Shared PyTorch runtime for dependent builds -### -FROM base AS build_pytorch_runtime -RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-13 130 --slave /usr/bin/g++ g++ /usr/bin/g++-13 -RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install,ro \ - pip install /install/*.whl - - -### -### TorchVision build -### -FROM build_pytorch_runtime AS build_torchvision -ARG PYTORCH_VISION_BRANCH -ARG PYTORCH_VISION_REPO -ARG USE_SCCACHE - -RUN git clone ${PYTORCH_VISION_REPO} vision -RUN cd vision && git checkout ${PYTORCH_VISION_BRANCH} \ - && if [ "$USE_SCCACHE" = "1" ]; then \ - export HIP_CLANG_PATH=/opt/sccache-wrappers \ - && export CMAKE_C_COMPILER_LAUNCHER=sccache \ - && export CMAKE_CXX_COMPILER_LAUNCHER=sccache; \ - fi \ - && python3 setup.py bdist_wheel --dist-dir=dist \ - && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi -RUN mkdir -p /app/install && cp /app/vision/dist/*.whl /app/install - - -### -### TorchAudio build -### -FROM build_pytorch_runtime AS build_torchaudio -ARG PYTORCH_AUDIO_BRANCH -ARG PYTORCH_AUDIO_REPO -ARG USE_SCCACHE - -RUN git clone ${PYTORCH_AUDIO_REPO} audio -RUN cd audio && git checkout ${PYTORCH_AUDIO_BRANCH} \ - && git submodule update --init --recursive -RUN cd audio && pip install -r requirements.txt -RUN cd audio \ - && if [ "$USE_SCCACHE" = "1" ]; then \ - export HIP_CLANG_PATH=/opt/sccache-wrappers \ - && export CMAKE_C_COMPILER_LAUNCHER=sccache \ - && export CMAKE_CXX_COMPILER_LAUNCHER=sccache; \ - fi \ - && python3 setup.py bdist_wheel --dist-dir=dist \ - && if [ "$USE_SCCACHE" = "1" ]; then sccache --show-stats; fi -RUN mkdir -p /app/install && cp /app/audio/dist/*.whl /app/install - - ### ### MORI Build ### FROM base AS build_mori ARG MORI_BRANCH ARG MORI_REPO -RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \ - pip install /install/*.whl RUN git clone ${MORI_REPO} RUN cd mori \ && git checkout ${MORI_BRANCH} \ && git submodule update --init --recursive \ - && python3 setup.py bdist_wheel --dist-dir=dist && ls /app/mori/dist/*.whl + && ln -sfn ${ROCM_PATH} /opt/rocm \ + && pip install "setuptools_scm>=6.2" \ + && CMAKE_PREFIX_PATH="${ROCM_PATH}:${ROCM_PATH}/lib/rocm_sysdeps" \ + python3 setup.py bdist_wheel --dist-dir=dist && ls /app/mori/dist/*.whl RUN mkdir -p /app/install && cp /app/mori/dist/*.whl /app/install - ### ### FlashAttention Build ### @@ -236,8 +220,6 @@ FROM base AS build_fa ARG FA_BRANCH ARG FA_REPO ARG USE_SCCACHE -RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \ - pip install /install/*.whl RUN git clone ${FA_REPO} RUN cd flash-attention \ && git checkout ${FA_BRANCH} \ @@ -258,12 +240,12 @@ FROM base AS build_aiter ARG AITER_BRANCH ARG AITER_REPO ARG USE_SCCACHE -RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \ - pip install /install/*.whl RUN git clone --recursive --branch ${AITER_BRANCH} ${AITER_REPO} RUN cd aiter \ && git submodule update --init --recursive \ && pip install -r requirements.txt +# AITER_USE_SYSTEM_TRITON keeps setup.py from resolving its own triton, which would +# displace the pinned one already installed in base. RUN pip install pyyaml && cd aiter \ && if [ "$USE_SCCACHE" = "1" ]; then \ export HIP_CLANG_PATH=/opt/sccache-wrappers \ @@ -276,58 +258,30 @@ RUN mkdir -p /app/install && cp /app/aiter/dist/*.whl /app/install ### -### Final Build +### Triton Build ### +FROM base AS build_triton +ARG TRITON_BRANCH +ARG TRITON_REPO +RUN git clone ${TRITON_REPO} +RUN cd triton \ + && git checkout ${TRITON_BRANCH} \ + && if [ ! -f setup.py ]; then cd python; fi \ + && python3 setup.py bdist_wheel --dist-dir=dist \ + && mkdir -p /app/install && cp dist/*.whl /app/install -# Wheel release stage - -# only includes dependencies used by wheel release pipeline -FROM base AS debs_wheel_release -RUN mkdir /app/debs -RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs -RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs -RUN --mount=type=bind,from=build_amdsmi,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs -RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs -RUN --mount=type=bind,from=build_torchvision,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs -RUN --mount=type=bind,from=build_torchaudio,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs -RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs - -# Full debs stage - includes Mori (used by Docker releases) -FROM base AS debs -RUN mkdir /app/debs -RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs -RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs -RUN --mount=type=bind,from=build_amdsmi,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs -RUN --mount=type=bind,from=build_pytorch,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs -RUN --mount=type=bind,from=build_torchvision,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs -RUN --mount=type=bind,from=build_torchaudio,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs -RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs -RUN --mount=type=bind,from=build_mori,src=/app/install/,target=/install \ - cp /install/*.whl /app/debs ### ### ROCr + CLR build ### -# Pull in updates to ROCr + clr from ROCm 7.2.4 and beyond. +# Brings in multi-stream perf improvements; drop after these arrive in an official release FROM base AS build_rocm_runtime ARG ROCM_SYSTEMS_REPO ARG ROCM_RUNTIME_COMMIT +# libgl-dev: rocclr does find_package(OpenGL REQUIRED) unconditionally and the wheel image has +# no GL headers. The link must not pick up a libGL DT_NEEDED, which is asserted after the build. RUN apt-get update \ - && apt-get install -y --no-install-recommends g++ \ - libelf-dev libdrm-dev libnuma-dev libdw-dev xxd \ + && apt-get install -y --no-install-recommends g++ libdw-dev xxd libgl-dev libglx-dev \ && rm -rf /var/lib/apt/lists/* RUN pip install --no-cache-dir CppHeaderParser RUN git init -q /src && cd /src \ @@ -339,20 +293,17 @@ RUN git init -q /src && cd /src \ && git sparse-checkout set projects/rocr-runtime projects/clr projects/hip shared cmake \ && git fetch --filter=blob:none --depth 1 origin ${ROCM_RUNTIME_COMMIT} \ && git checkout -q FETCH_HEAD -# ROCr's trap_handler and blit_shaders call find_package(Clang/LLVM REQUIRED) only to get -# IMPORTED executables. The ROCm dev image ships the binaries but no CMake package config, so -# supply one pointing at them. -RUN mkdir -p /opt/rocm-cmake-shim \ - && printf 'if(NOT TARGET clang)\n add_executable(clang IMPORTED GLOBAL)\n set_target_properties(clang PROPERTIES IMPORTED_LOCATION "/opt/rocm/llvm/bin/clang")\nendif()\nset(Clang_PACKAGE_VERSION "rocm-image-shim")\n' > /opt/rocm-cmake-shim/ClangConfig.cmake \ - && printf 'if(NOT TARGET llvm-objcopy)\n add_executable(llvm-objcopy IMPORTED GLOBAL)\n set_target_properties(llvm-objcopy PROPERTIES IMPORTED_LOCATION "/opt/rocm/llvm/bin/llvm-objcopy")\nendif()\nset(LLVM_FOUND TRUE)\nset(LLVM_PACKAGE_VERSION "rocm-image-shim")\n' > /opt/rocm-cmake-shim/LLVMConfig.cmake -# ROCr first, and installed into /opt/rocm, because the clr build below needs its headers +# libelf is taken from the SDK's own vendored sysdeps rather than apt -- the wheel image has no +# /usr/include/libelf.h -- so ROCr links against the copy the wheel itself was built against. +ENV SYSDEPS="$SDK_DEV/lib/rocm_sysdeps" +# ROCr first, and installed into $SDK_DEV, because the clr build below needs its headers. RUN cd /src/projects/rocr-runtime \ && cmake -B build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH=/opt/rocm \ - -DCMAKE_INSTALL_PREFIX=/opt/rocm \ - -DClang_DIR=/opt/rocm-cmake-shim \ - -DLLVM_DIR=/opt/rocm-cmake-shim \ + -DCMAKE_PREFIX_PATH="$SDK_DEV;$SYSDEPS" \ + -DCMAKE_INSTALL_PREFIX="$SDK_DEV" \ + -DLIBELF_INCLUDE_DIRS="$SYSDEPS/include" \ + -DLIBELF_LIBRARIES="$SYSDEPS/lib/libelf.so" \ && cmake --build build --parallel 16 \ && cmake --install build \ && cmake --install build --prefix /rocr-install --strip @@ -361,12 +312,12 @@ RUN cd /src/projects/clr \ -DCMAKE_BUILD_TYPE=Release \ -DCLR_BUILD_HIP=ON \ -DCLR_BUILD_OCL=OFF \ + -DROCM_KPACK_ENABLED=ON \ -DHIP_COMMON_DIR=/src/projects/hip \ - -DROCM_PATH=/opt/rocm \ - -DCMAKE_PREFIX_PATH=/opt/rocm \ - -DCMAKE_INSTALL_PREFIX=/opt/rocm \ - -DLLVM_DIR=/opt/rocm-cmake-shim \ - -DHIP_LLVM_ROOT=/opt/rocm/llvm \ + -DROCM_PATH="$SDK_DEV" \ + -DCMAKE_PREFIX_PATH="$SDK_DEV;$SYSDEPS" \ + -DCMAKE_INSTALL_PREFIX="$SDK_DEV" \ + -DHIP_LLVM_ROOT="$SDK_DEV/lib/llvm" \ && cmake --build build --parallel 16 \ && cmake --install build --prefix /clr-install --strip RUN mkdir -p /staging/lib \ @@ -374,82 +325,115 @@ RUN mkdir -p /staging/lib \ && cp -P /clr-install/lib/libamdhip64.so* /staging/lib/ \ && ls -l /staging/lib -# rocprofiler-sdk 1.3.2 fixes torch 2.12 profiler on ROCm 7.2.3 -FROM base AS build_rocprofiler_sdk -ARG ROCM_SYSTEMS_REPO -ARG ROCPROFILER_SDK_COMMIT -ARG USE_SCCACHE -RUN apt-get update \ - && apt-get install -y --no-install-recommends libdw-dev libelf-dev libsqlite3-dev libzstd-dev \ - && rm -rf /var/lib/apt/lists/* -RUN git init -q /src && cd /src \ - && git remote add origin ${ROCM_SYSTEMS_REPO} \ - && git config core.sparseCheckout true \ - && git config remote.origin.promisor true \ - && git config remote.origin.partialclonefilter blob:none \ - && git sparse-checkout init --cone \ - && git sparse-checkout set projects/rocprofiler-sdk shared cmake \ - && git fetch --filter=blob:none --depth 1 origin ${ROCPROFILER_SDK_COMMIT} \ - && git checkout -q FETCH_HEAD -# Patch the ROCm 7.2.3 build of rocprofiler-sdk to include queue-interposition fixes that are -# absent from the pinned 7.14 build. This resolves hangs on torch.profiler on ROCm. -# TODO: Drop these once ${ROCPROFILER_SDK_COMMIT} is bumped to a build that already -# contains them, such as ROCm 10.0. See https://github.com/vllm-project/vllm/issues/54087. -ARG ROCPROFILER_SDK_PR_7796=fae7fd57ffb8a76e811d6410b8a0cee1e94d710a -ARG ROCPROFILER_SDK_PR_7924=f8db22adbe17686a9b05ae03367dd1ec73011653 -RUN cd /src \ - && HSA_PATHS='projects/rocprofiler-sdk/source/lib/rocprofiler-sdk/hsa/*' \ - && curl -fsSL -o /tmp/qi-7796.patch "${ROCM_SYSTEMS_REPO%.git}/commit/${ROCPROFILER_SDK_PR_7796}.patch" \ - && curl -fsSL -o /tmp/qi-7924.patch "${ROCM_SYSTEMS_REPO%.git}/commit/${ROCPROFILER_SDK_PR_7924}.patch" \ - && git apply -p1 --include="$HSA_PATHS" --verbose /tmp/qi-7796.patch \ - && git apply -p1 --include="$HSA_PATHS" --verbose /tmp/qi-7924.patch -RUN cd /src/projects/rocprofiler-sdk \ - && if [ "$USE_SCCACHE" = "1" ]; then \ - export CMAKE_C_COMPILER_LAUNCHER=sccache \ - && export CMAKE_CXX_COMPILER_LAUNCHER=sccache; \ - fi \ - && cmake -B build -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_PREFIX_PATH=/opt/rocm \ - -DCMAKE_INSTALL_PREFIX=/opt/rocm \ - -DROCPROFILER_BUILD_TESTS=OFF \ - -DROCPROFILER_BUILD_SAMPLES=OFF \ - -DROCPROFILER_BUILD_DOCS=OFF \ - && cmake --build build --parallel "$(nproc)" \ - && cmake --install build --prefix /staging --strip + +### +### Final Build +### + +# Wheel release stage - +# only includes dependencies used by wheel release pipeline +FROM base AS debs_wheel_release +RUN mkdir /app/debs +RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +# The wheel index also serves a torch repacked to require our triton instead of +# the one TheRock built it with, and amdsmi, which TheRock ships only as sources +# inside rocm-sdk-core. +ARG TORCH_VERSION +ARG ROCM_RELEASE_WHEELS_MULTIARCH_URL +ARG ROCM_NIGHTLY_WHEELS_MULTIARCH_URL +COPY tools/vllm-rocm/therock_wheels.py /tmp/therock_wheels.py +# regex and pybase64 aren't installed here; fall back to the stdlib modules. +RUN sed -i -e 's/import regex as re/import re/' \ + -e 's/import pybase64 as base64/import base64/' /tmp/therock_wheels.py \ + && pip download --no-deps --dest /tmp/torch-dl \ + --index-url ${ROCM_RELEASE_WHEELS_MULTIARCH_URL} \ + --extra-index-url ${ROCM_NIGHTLY_WHEELS_MULTIARCH_URL} \ + "torch==${TORCH_VERSION}" \ + && TRITON_VER=$(basename /app/debs/triton-*.whl | cut -d- -f2) \ + && python3 /tmp/therock_wheels.py repack-torch /tmp/torch-dl/torch-*.whl \ + --triton-version "${TRITON_VER}" --out-dir /app/debs \ + && python3 /tmp/therock_wheels.py patch-amdsmi "${SDK_CORE}/share/amd_smi" \ + --out-dir /tmp/amd_smi \ + && pip wheel --no-deps --wheel-dir /app/debs /tmp/amd_smi \ + && rm -rf /tmp/torch-dl /tmp/amd_smi /tmp/therock_wheels.py \ + && ls -l /app/debs + +FROM base AS debs +RUN mkdir /app/debs +RUN --mount=type=bind,from=build_triton,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_fa,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_aiter,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs +RUN --mount=type=bind,from=build_mori,src=/app/install/,target=/install \ + cp /install/*.whl /app/debs FROM base AS final -RUN --mount=type=bind,from=debs,src=/app/debs,target=/install \ - pip install /install/*.whl +ARG TRITON_VERSION -COPY --from=build_rocprofiler_sdk /staging/ /opt/rocm/ +# torch 2.12 implicitly attaches rocprofiler-sdk; its queue interposition hangs at HSA +# teardown, so disable it. TODO: drop once ROCm's stock rocprofiler carries the fix. ENV ROCPROFILER_QUEUE_INTERPOSITION=0 -COPY --from=build_rocm_runtime /staging/ /opt/rocm/ -# Hard link the patched library onto every stock file name so that both the -# SONAME lookup and the hard coded paths resolve to it. +COPY --from=build_rocm_runtime /staging/lib/ ${SDK_DEV}/lib/ +# Hard link the patched library onto every stock file name, in both SDK trees, so that the +# SONAME lookup and the hard coded paths all resolve to it. Hard links rather than symlinks +# because ldconfig rewrites SONAME symlinks but cannot re-point an inode. $SDK_CORE keeps its +# own libhsa-runtime64 copy, which is on LD_LIBRARY_PATH and in ld.so.conf; relinking it is +# belt and braces rather than strictly required, but it leaves no stock ROCr in the image. RUN set -eux; \ - cd /opt/rocm/lib; \ + cd "$SDK_DEV/lib"; \ ours_hsa="$(readlink -f libhsa-runtime64.so.1)"; \ ours_hip="$(readlink -f libamdhip64.so.7)"; \ - for f in libhsa-runtime64.so.1.*; do \ - [ "$f" = "${ours_hsa##*/}" ] || ln -f "$ours_hsa" "$f"; \ - done; \ - for f in libamdhip64.so.7.*; do \ - [ "$f" = "${ours_hip##*/}" ] || ln -f "$ours_hip" "$f"; \ + for f in "$SDK_DEV"/lib/libhsa-runtime64.so.1* "$SDK_CORE"/lib/libhsa-runtime64.so.1* \ + "$SDK_DEV"/lib/libamdhip64.so.7* "$SDK_CORE"/lib/libamdhip64.so.7*; do \ + [ -f "$f" ] && [ ! -L "$f" ] || continue; \ + case "${f##*/}" in libhsa*) ours="$ours_hsa";; *) ours="$ours_hip";; esac; \ + [ "$f" -ef "$ours" ] || ln -f "$ours" "$f"; \ done RUN ldconfig +# This is to prevent resolvers from checking METADATA and RECORD and clobbering +# the Triton source build we installed. Constraints files are unable to help +# with this. However, these entries are optional and can be blanked which +# solves the problem. +RUN TORCH_DIST_INFO=$(ls -d ${SITE_PACKAGES}/torch-*.dist-info) \ + && grep -q '^Requires-Dist: triton' "$TORCH_DIST_INFO/METADATA" \ + && sed -i '/^Requires-Dist: triton/d' "$TORCH_DIST_INFO/METADATA" \ + && sed -i "s|^\(${TORCH_DIST_INFO##*/}/METADATA\),.*|\1,,|" "$TORCH_DIST_INFO/RECORD" + +RUN --mount=type=bind,from=debs,src=/app/debs,target=/install \ + pip install /install/*.whl \ + && ! pip check 2>&1 | grep -iE '^(torch|triton) ' \ + && test "$(pip show triton | awk '/^Version:/ {print $2}')" = "${TRITON_VERSION}" \ + # Create a constraints file to protect the versions we need, especially + # triton, requirements will try to reinstall repeatedly. + && pip list --format=freeze \ + | grep -E '^(torch|triton|torchaudio|torchvision|pybind11)==' > /etc/rocm-constraints.txt \ + && echo "ROCm constraints:" && cat /etc/rocm-constraints.txt \ + && test "$(wc -l < /etc/rocm-constraints.txt)" -eq 5 \ + && grep -q '^pybind11==3\.0\.' /etc/rocm-constraints.txt \ + && python3 -c "import torch; assert torch.version.hip, f'non-ROCm torch: {torch.__version__}'; print(f'torch={torch.__version__}')" \ + && python3 -c "import triton; print(f'triton={triton.__version__}')" \ + && pip list 2>/dev/null | grep -iE '^(torch|triton) ' + ARG BASE_IMAGE +ARG PYTHON_VERSION +ARG ROCM_SDK_VERSION +ARG TORCH_VERSION +ARG TORCHAUDIO_VERSION +ARG TORCHVISION_VERSION +ARG PYTORCH_ROCM_ARCH +ARG AITER_ROCM_ARCH ARG TRITON_BRANCH ARG TRITON_REPO -ARG PYTORCH_BRANCH -ARG PYTORCH_VISION_BRANCH -ARG PYTORCH_REPO -ARG PYTORCH_VISION_REPO -ARG PYTORCH_AUDIO_BRANCH -ARG PYTORCH_AUDIO_REPO ARG FA_BRANCH ARG FA_REPO ARG AITER_BRANCH @@ -457,23 +441,36 @@ ARG AITER_REPO ARG MORI_BRANCH ARG MORI_REPO ARG ROCM_SYSTEMS_REPO -ARG ROCPROFILER_SDK_COMMIT ARG ROCM_RUNTIME_COMMIT +ARG TRITON_VERSION +# Record hashes for Triton and pybind11 so we can verify later. RUN echo "BASE_IMAGE: ${BASE_IMAGE}" > /app/versions.txt \ - && echo "ROCM_SYSTEMS_REPO: ${ROCM_SYSTEMS_REPO}" >> /app/versions.txt \ - && echo "ROCPROFILER_SDK_COMMIT: ${ROCPROFILER_SDK_COMMIT}" >> /app/versions.txt \ - && echo "ROCM_RUNTIME_COMMIT: ${ROCM_RUNTIME_COMMIT}" >> /app/versions.txt \ + && echo "PYTHON_VERSION: ${PYTHON_VERSION}" >> /app/versions.txt \ + && echo "ROCM_SDK_VERSION: ${ROCM_SDK_VERSION}" >> /app/versions.txt \ + && echo "TORCH_VERSION: ${TORCH_VERSION}" >> /app/versions.txt \ + && echo "TORCHAUDIO_VERSION: ${TORCHAUDIO_VERSION}" >> /app/versions.txt \ + && echo "TORCHVISION_VERSION: ${TORCHVISION_VERSION}" >> /app/versions.txt \ + && echo "PYTORCH_ROCM_ARCH: ${PYTORCH_ROCM_ARCH}" >> /app/versions.txt \ + && echo "AITER_ROCM_ARCH: ${AITER_ROCM_ARCH}" >> /app/versions.txt \ && echo "TRITON_BRANCH: ${TRITON_BRANCH}" >> /app/versions.txt \ && echo "TRITON_REPO: ${TRITON_REPO}" >> /app/versions.txt \ - && echo "PYTORCH_BRANCH: ${PYTORCH_BRANCH}" >> /app/versions.txt \ - && echo "PYTORCH_VISION_BRANCH: ${PYTORCH_VISION_BRANCH}" >> /app/versions.txt \ - && echo "PYTORCH_REPO: ${PYTORCH_REPO}" >> /app/versions.txt \ - && echo "PYTORCH_VISION_REPO: ${PYTORCH_VISION_REPO}" >> /app/versions.txt \ - && echo "PYTORCH_AUDIO_BRANCH: ${PYTORCH_AUDIO_BRANCH}" >> /app/versions.txt \ - && echo "PYTORCH_AUDIO_REPO: ${PYTORCH_AUDIO_REPO}" >> /app/versions.txt \ && echo "FA_BRANCH: ${FA_BRANCH}" >> /app/versions.txt \ && echo "FA_REPO: ${FA_REPO}" >> /app/versions.txt \ && echo "AITER_BRANCH: ${AITER_BRANCH}" >> /app/versions.txt \ && echo "AITER_REPO: ${AITER_REPO}" >> /app/versions.txt \ && echo "MORI_BRANCH: ${MORI_BRANCH}" >> /app/versions.txt \ - && echo "MORI_REPO: ${MORI_REPO}" >> /app/versions.txt + && echo "MORI_REPO: ${MORI_REPO}" >> /app/versions.txt \ + && echo "ROCM_SYSTEMS_REPO: ${ROCM_SYSTEMS_REPO}" >> /app/versions.txt \ + && echo "ROCM_RUNTIME_COMMIT: ${ROCM_RUNTIME_COMMIT}" >> /app/versions.txt \ + # Fingerprint our triton so Dockerfile.rocm can prove it survived the build. + && echo "TRITON_VERSION: $(pip show triton | awk '/^Version:/ {print $2}')" >> /app/versions.txt \ + && echo "TRITON_SHA256: $(find $(python3 -c 'import triton, os; print(os.path.dirname(triton.__file__))') \ + -type f \( -name '*.py' -o -name '*.so' \) | sort | xargs sha256sum | sha256sum | cut -d' ' -f1)" \ + >> /app/versions.txt \ + && echo "PYBIND11_VERSION: $(pip show pybind11 | awk '/^Version:/ {print $2}')" >> /app/versions.txt \ + && echo "PYBIND11_INTERNALS_VERSION: $(grep -rhE '^#[[:space:]]*define PYBIND11_INTERNALS_VERSION' \ + $(python3 -c 'import pybind11; print(pybind11.get_include())') | awk '{print $NF}')" \ + >> /app/versions.txt \ + && echo "PYBIND11_SHA256: $(find $(python3 -c 'import pybind11; print(pybind11.get_include())') \ + -type f -name '*.h' | sort | xargs sha256sum | sha256sum | cut -d' ' -f1)" \ + >> /app/versions.txt diff --git a/docker/ci-rocm.hcl b/docker/ci-rocm.hcl index 1794b8468a..c251641452 100644 --- a/docker/ci-rocm.hcl +++ b/docker/ci-rocm.hcl @@ -83,10 +83,6 @@ variable "UCX_BRANCH" { default = "" } -variable "ROCSHMEM_BRANCH" { - default = "" -} - variable "DEEPEP_BRANCH" { default = "" } @@ -95,10 +91,6 @@ variable "NIXL_CACHE_KEY" { default = "" } -variable "ROCSHMEM_CACHE_KEY" { - default = "" -} - variable "DEEPEP_CACHE_KEY" { default = "" } @@ -236,7 +228,7 @@ function "get_cache_to_rocm_rust" { ]) } -# Cache functions for upstream dependency stages (NIXL/UCX, ROCShmem, DeepEP). +# Cache functions for upstream dependency stages (NIXL/UCX, DeepEP). # These stages are pinned to specific upstream commit hashes, so cache keys use # those hashes rather than the Buildkite commit. This means the cache persists # across all vLLM commits as long as the upstream dependency pins don't change. @@ -245,8 +237,7 @@ function "get_cache_from_rocm_deps" { params = [] result = compact([ NIXL_CACHE_KEY != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:nixl-rocm-${NIXL_CACHE_KEY}" : (NIXL_BRANCH != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:nixl-rocm-${NIXL_BRANCH}-ucx-${UCX_BRANCH}" : ""), - ROCSHMEM_CACHE_KEY != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rocshmem-rocm-${ROCSHMEM_CACHE_KEY}" : (ROCSHMEM_BRANCH != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rocshmem-rocm-${ROCSHMEM_BRANCH}" : ""), - DEEPEP_CACHE_KEY != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:deepep-rocm-${DEEPEP_CACHE_KEY}" : (DEEPEP_BRANCH != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:deepep-rocm-${DEEPEP_BRANCH}-rocshmem-${ROCSHMEM_BRANCH}" : ""), + DEEPEP_CACHE_KEY != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:deepep-rocm-${DEEPEP_CACHE_KEY}" : (DEEPEP_BRANCH != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:deepep-rocm-${DEEPEP_BRANCH}" : ""), ]) } @@ -257,17 +248,10 @@ function "get_cache_to_rocm_nixl" { ]) } -function "get_cache_to_rocm_rocshmem" { - params = [] - result = compact([ - ROCSHMEM_CACHE_KEY != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rocshmem-rocm-${ROCSHMEM_CACHE_KEY},mode=min" : (ROCSHMEM_BRANCH != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:rocshmem-rocm-${ROCSHMEM_BRANCH},mode=min" : ""), - ]) -} - function "get_cache_to_rocm_deepep" { params = [] result = compact([ - DEEPEP_CACHE_KEY != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:deepep-rocm-${DEEPEP_CACHE_KEY},mode=min" : (DEEPEP_BRANCH != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:deepep-rocm-${DEEPEP_BRANCH}-rocshmem-${ROCSHMEM_BRANCH},mode=min" : ""), + DEEPEP_CACHE_KEY != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:deepep-rocm-${DEEPEP_CACHE_KEY},mode=min" : (DEEPEP_BRANCH != "" ? "type=registry,ref=${DOCKERHUB_CACHE_REPO}:deepep-rocm-${DEEPEP_BRANCH},mode=min" : ""), ]) } @@ -385,14 +369,6 @@ target "nixl-rocm-ci" { output = ["type=cacheonly"] } -target "rocshmem-rocm-ci" { - inherits = ["_common-rocm", "_ci-rocm"] - target = "build_rocshmem" - cache-from = get_cache_from_rocm_deps() - cache-to = get_cache_to_rocm_rocshmem() - output = ["type=cacheonly"] -} - target "deepep-rocm-ci" { inherits = ["_common-rocm", "_ci-rocm"] target = "build_deepep" @@ -418,7 +394,7 @@ target "ci-base-rocm-ci" { CI_BASE_IMAGE_TAG != "" ? "type=registry,ref=${CI_BASE_IMAGE_TAG}" : "", CI_BASE_TRUSTED_CONTENT_REF != "" ? "type=registry,ref=${CI_BASE_TRUSTED_CONTENT_REF}" : "", ]), - # Import upstream dependency caches so NIXL/ROCShmem/DeepEP stages + # Import upstream dependency caches so NIXL/DeepEP stages # are cache hits even when ci_base itself needs rebuilding. get_cache_from_rocm_deps(), ) @@ -431,5 +407,5 @@ target "ci-base-rocm-ci" { # Group for ci_base builds -- exports dependency stage caches alongside the # ci_base image so future rebuilds can reuse them independently. group "ci-base-rocm-ci-with-deps" { - targets = ["nixl-rocm-ci", "rocshmem-rocm-ci", "deepep-rocm-ci", "ci-base-rocm-ci"] + targets = ["nixl-rocm-ci", "deepep-rocm-ci", "ci-base-rocm-ci"] } diff --git a/docker/docker-bake-rocm.hcl b/docker/docker-bake-rocm.hcl index ab523a296a..2eb1d5a430 100644 --- a/docker/docker-bake-rocm.hcl +++ b/docker/docker-bake-rocm.hcl @@ -65,10 +65,6 @@ variable "UCX_BRANCH" { default = "" } -variable "ROCSHMEM_BRANCH" { - default = "" -} - variable "DEEPEP_BRANCH" { default = "" } diff --git a/docs/getting_started/installation/gpu.rocm.inc.md b/docs/getting_started/installation/gpu.rocm.inc.md index 28e74b670c..d8debbab3e 100644 --- a/docs/getting_started/installation/gpu.rocm.inc.md +++ b/docs/getting_started/installation/gpu.rocm.inc.md @@ -1,7 +1,7 @@ --8<-- [start:installation] -vLLM supports AMD GPUs with ROCm 6.3 or above. Pre-built wheels are available for ROCm 7.0 and ROCm 7.2.1. +vLLM supports AMD GPUs with ROCm 6.3 or above. Pre-built wheels are available for ROCm 10.0 (the default) and ROCm 7.2. #### Prebuilt Wheels @@ -9,6 +9,8 @@ vLLM supports AMD GPUs with ROCm 6.3 or above. Pre-built wheels are available fo | ------------ | -------------- | ------------ | ----------------- | ------------------ | | `rocm700` | 3.12 | 7.0 | >= 2.35 | `0.14.0` to `0.18.0` | | `rocm721` | 3.12 | 7.2.1 | >= 2.35 | Nightly releases after commit `171775f306a333a9cf105bfd533bf3e113d401d9` | +| `rocm72` | 3.12 | 7.2 | >= 2.39 | Nightly releases on the legacy ROCm 7.2 stack | +| `rocm100` | 3.12 | 10.0 | >= 2.35 | Nightly releases (default) | --8<-- [end:installation] --8<-- [start:requirements] @@ -40,12 +42,17 @@ If you need a different ROCm version or want to use an existing PyTorch installa source .venv/bin/activate ``` -To install the latest version of vLLM for Python 3.12, ROCm 7.0 and `glibc >= 2.35`. +To install the latest version of vLLM for Python 3.12, ROCm 10.0 and `glibc >= 2.35`: ```bash uv pip install vllm --extra-index-url https://wheels.vllm.ai/rocm/ --upgrade ``` +!!! tip + The ROCm 10.0 wheels install the ROCm SDK from pip ([TheRock](https://github.com/ROCm/TheRock)), including the GPU kernels for every supported architecture. + + Wheels for the legacy ROCm 7.2 stack, which use a system ROCm installation, remain available at `https://wheels.vllm.ai/rocm/${VLLM_VERSION}/rocm72`. + !!! tip You can find out about which ROCm version the latest vLLM supports by checking the `vllm` package in index in extra-index-url at [https://wheels.vllm.ai/rocm/vllm](https://wheels.vllm.ai/rocm/vllm) . @@ -195,9 +202,9 @@ install it without re-running the full installation: and place it in `vllm/vllm-rs`. If you see an error about a wheel not being found, the wheel for your base -commit and ROCm patch version might not be available. Check the available -variants under `https://wheels.vllm.ai/rocm//`. For example, ROCm 7.2.1 -uses the `rocm721` variant. +commit and ROCm version might not be available. Check the available +variants under `https://wheels.vllm.ai/rocm//`. Variants are named by ROCm +major and minor version, e.g. ROCm 7.2.3 uses the `rocm72` variant. There are more environment variables to control the behavior of Python-only build: @@ -208,7 +215,7 @@ build: - `VLLM_PRECOMPILED_WHEEL_COMMIT`: override the full commit hash used to download the pre-compiled wheel. - `VLLM_PRECOMPILED_WHEEL_VARIANT`: specify the ROCm variant subdirectory, e.g., - `rocm700` or `rocm721`. If not specified, the variant is auto-detected based + `rocm72` or `rocm100`. If not specified, the variant is auto-detected based on your system's ROCm version. An explicitly specified variant must match the detected environment. @@ -226,23 +233,34 @@ You can find more information about vLLM's wheels in #### Full build (with compilation) {#full-build} !!! tip - - If you found that the following installation step does not work for you, please refer to [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base). Dockerfile is a form of installation steps. + - The steps below follow [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base), which pins the validated version of every component. If a step does not work for you, refer to it; a Dockerfile is a form of installation steps. + - The legacy ROCm 7.2 stack uses a system ROCm installation instead; its steps are in [docker/Dockerfile.rocm_72_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_72_base). -0. Install prerequisites (skip if you are already in an environment/docker with the following installed): +0. Install the ROCm SDK and PyTorch (skip if you are already in an environment or docker image with them installed, e.g. `vllm/vllm-openai-rocm:base-nightly`). - - [ROCm](https://rocm.docs.amd.com/en/latest/deploy/linux/index.html) - - [PyTorch](https://pytorch.org/) - - For installing PyTorch, you can start from a fresh docker image, e.g, `rocm/pytorch:rocm7.0_ubuntu22.04_py3.10_pytorch_release_2.8.0`, `rocm/pytorch-nightly`. If you are using docker image, you can skip to Step 3. - - Alternatively, you can install PyTorch using PyTorch wheels. You can check PyTorch installation guide in PyTorch [Getting Started](https://pytorch.org/get-started/locally/). Example: + ROCm 10.0 is installed from pip ([TheRock](https://github.com/ROCm/TheRock)), together with PyTorch and the GPU kernels for your GPU's architecture. In a Python 3.12 environment: ```bash - # Install PyTorch - pip uninstall torch -y - pip install --no-cache-dir torch torchvision --index-url https://download.pytorch.org/whl/nightly/rocm7.0 + # Your GPU's architecture, e.g. gfx942 for MI300 or gfx950 for MI350 + export GPU_ARCH=$(uvx --from rocm-bootstrap rocm-bootstrap-detect --unique) + export ROCM_INDEX=https://stable.repo.amd.com/rocm/whl-next/ + + pip install --index-url ${ROCM_INDEX} \ + "rocm[libraries,devel,device-${GPU_ARCH}]==10.0.0" \ + "torch[device-${GPU_ARCH}]==2.12.0+rocm10.0.0" \ + "torchvision[device-${GPU_ARCH}]==0.27.0+rocm10.0.0" + rocm-sdk init + + # Point builds at the pip-installed ROCm SDK + export ROCM_PATH=$(rocm-sdk path --root) + export ROCM_HOME=${ROCM_PATH} HIP_PATH=${ROCM_PATH} + export PATH=$(rocm-sdk path --bin):${PATH} ``` + !!! note + - The validated `ROCM_SDK_VERSION`, `TORCH_VERSION` and `TORCHVISION_VERSION` can be found in [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base). + - To support several GPU architectures, add a `device-` extra for each of them. + 1. Install [Triton for ROCm](https://github.com/ROCm/triton.git) Install ROCm's Triton following the instructions from [ROCm/triton](https://github.com/ROCm/triton.git) @@ -252,8 +270,7 @@ You can find more information about vLLM's wheels in pip uninstall -y triton git clone https://github.com/ROCm/triton.git cd triton - # git checkout $TRITON_BRANCH - git checkout f9e5bf54 + git checkout $TRITON_BRANCH if [ ! -f setup.py ]; then cd python; fi python3 setup.py install cd ../.. @@ -265,17 +282,14 @@ You can find more information about vLLM's wheels in 2. Optionally, if you choose to use CK flash attention, you can install [flash attention for ROCm](https://github.com/Dao-AILab/flash-attention.git) - Install ROCm's flash attention (v2.8.0) following the instructions from [ROCm/flash-attention](https://github.com/Dao-AILab/flash-attention#amd-rocm-support) - - For example, for ROCm 7.0, suppose your gfx arch is `gfx942`. To get your gfx architecture, run `rocminfo |grep gfx`. + Install ROCm's flash attention following the instructions from [ROCm/flash-attention](https://github.com/Dao-AILab/flash-attention#amd-rocm-support) ```bash git clone https://github.com/Dao-AILab/flash-attention.git cd flash-attention - # git checkout $FA_BRANCH - git checkout 0e60e394 + git checkout $FA_BRANCH git submodule update --init - GPU_ARCHS="gfx942" python3 setup.py install + GPU_ARCHS="${GPU_ARCH}" python3 setup.py install cd .. ``` @@ -290,7 +304,8 @@ You can find more information about vLLM's wheels in cd aiter git checkout $AITER_BRANCH_OR_COMMIT git submodule sync; git submodule update --init --recursive - python3 setup.py develop + # Keep the Triton installed in step 1 + AITER_USE_SYSTEM_TRITON=1 python3 setup.py develop ``` !!! note @@ -300,32 +315,34 @@ You can find more information about vLLM's wheels in 4. Optionally, if you want to use MORI for EP or PD disaggregation, you can install [MORI](https://github.com/ROCm/mori) using the following steps: ```bash + pip install "setuptools_scm>=6.2" git clone https://github.com/ROCm/mori.git cd mori git checkout $MORI_BRANCH_OR_COMMIT git submodule sync; git submodule update --init --recursive - MORI_GPU_ARCHS="gfx942;gfx950" python3 setup.py install + # The ROCm SDK keeps its CMake configs for system dependencies (e.g. NUMA) under rocm_sysdeps + CMAKE_PREFIX_PATH="${ROCM_PATH}:${ROCM_PATH}/lib/rocm_sysdeps" \ + MORI_GPU_ARCHS="gfx942;gfx950" python3 setup.py install ``` !!! note - You will need to config the `$MORI_BRANCH_OR_COMMIT` for your purpose. - The validated `$MORI_BRANCH_OR_COMMIT` can be found in the [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base). -5. Build vLLM. For example, vLLM on ROCM 7.0 can be built with the following steps: +5. Build vLLM. For example, vLLM on ROCm 10.0 can be built with the following steps: ???+ console "Commands" ```bash pip install --upgrade pip - # Build & install AMD SMI - pip install /opt/rocm/share/amd_smi + # Install AMD SMI, which the ROCm SDK ships as sources + SDK_CORE=$(python3 -c "import _rocm_sdk_core, os; print(os.path.dirname(_rocm_sdk_core.__file__))") + pip install ${SDK_CORE}/share/amd_smi + # It loads the SDK's libamd_smi relative to its own directory + export PYTHONPATH=${SDK_CORE}/share/amd_smi # Install dependencies - pip install --upgrade numba \ - scipy \ - huggingface-hub \ - setuptools_scm pip install -r requirements/rocm.txt # To build for a single architecture (e.g., MI300) for faster installation (recommended): @@ -352,8 +369,9 @@ You can find more information about vLLM's wheels in vLLM offers official Docker images for deployment. The images can be used to run OpenAI compatible server and are available on Docker Hub as [vllm/vllm-openai-rocm](https://hub.docker.com/r/vllm/vllm-openai-rocm/tags). -- `vllm/vllm-openai-rocm:latest` — stable release -- `vllm/vllm-openai-rocm:nightly` — preview build from the latest development branch, use this if you want the latest features and fixes +- `vllm/vllm-openai-rocm:latest` — stable release, built on ROCm 10.0 (also tagged `latest-rocm100`) +- `vllm/vllm-openai-rocm:nightly` — preview build from the latest development branch on ROCm 10.0 (also tagged `nightly-rocm100`), use this if you want the latest features and fixes +- `vllm/vllm-openai-rocm:latest-rocm72` / `nightly-rocm72` — the same builds on the legacy ROCm 7.2 stack, kept for a transition period ```bash docker run --rm \ @@ -409,7 +427,7 @@ You can build and run vLLM from source via the provided [docker/Dockerfile.rocm] ??? info "(Optional) Build an image with ROCm software stack" Build a docker image from [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base) which setup ROCm software stack needed by the vLLM. - **This step is optional as this rocm_base image is usually prebuilt and store at [Docker Hub](https://hub.docker.com/r/rocm/vllm-dev) under tag `rocm/vllm-dev:base` to speed up user experience.** + **This step is optional as this rocm_base image is usually prebuilt and store at [Docker Hub](https://hub.docker.com/r/rocm/vllm-dev) under tag `rocm/vllm-dev:base` to speed up user experience.** For the legacy ROCm 7.2 stack, the base image is built from [docker/Dockerfile.rocm_72_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_72_base) and published as `vllm/vllm-openai-rocm:base-nightly-rocm72`. If you choose to build this rocm_base image yourself, the steps are as follows. It is important that the user kicks off the docker build using buildkit. Either the user put `DOCKER_BUILDKIT=1` as environment variable when calling docker build command, or the user needs to set up buildkit in the docker daemon configuration `/etc/docker/daemon.json` as follows and restart the daemon: @@ -422,7 +440,7 @@ You can build and run vLLM from source via the provided [docker/Dockerfile.rocm] } ``` - To build vllm on ROCm 7.0 for MI200 and MI300 series, you can use the default: + To build vllm on ROCm 10.0 for all supported GPUs, you can use the default: ```bash DOCKER_BUILDKIT=1 docker build \ @@ -441,15 +459,15 @@ It is important that the user kicks off the docker build using buildkit. Either } ``` -[docker/Dockerfile.rocm](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm) uses ROCm 7.0 by default, but also supports ROCm 5.7, 6.0, 6.1, 6.2, 6.3, and 6.4, in older vLLM branches. +[docker/Dockerfile.rocm](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm) uses ROCm 10.0 (installed from TheRock wheels) by default. The legacy ROCm 7.2 stack is available as [docker/Dockerfile.rocm_72](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_72) with [docker/Dockerfile.rocm_72_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_72_base); older vLLM branches support ROCm 5.7 through 7.0. It provides flexibility to customize the build of docker image using the following arguments: -- `BASE_IMAGE`: specifies the base image used when running `docker build`. The default value `rocm/vllm-dev:base` is an image published and maintained by AMD. It is being built using [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base) +- `BASE_IMAGE`: specifies the base image used when running `docker build`. The default value `rocm/vllm-dev:base` is an image published and maintained by AMD. It is being built using [docker/Dockerfile.rocm_base](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_base). For [docker/Dockerfile.rocm_72](https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.rocm_72), the default is `vllm/vllm-openai-rocm:base-nightly-rocm72` - `ARG_PYTORCH_ROCM_ARCH`: Allows to override the gfx architecture values from the base docker image Their values can be passed in when running `docker build` with `--build-arg` options. -To build vllm on ROCm 7.0 for MI200 and MI300 series, you can use the default (which build a docker image with `vllm serve` as entrypoint): +To build vllm on ROCm 10.0 for all supported GPUs, you can use the default (which build a docker image with `vllm serve` as entrypoint): ```bash DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.rocm -t vllm/vllm-openai-rocm . diff --git a/setup.py b/setup.py index 83b59e303a..cf99ec3aaa 100644 --- a/setup.py +++ b/setup.py @@ -645,8 +645,10 @@ class precompiled_wheel_utils: @staticmethod def rocm_version_to_variant(rocm_version: str) -> str: - """Convert a ROCm version string to a wheel variant, e.g. 7.2.3 -> rocm723.""" - return "rocm" + rocm_version.replace(".", "") + """Convert a ROCm version string to a wheel variant from its major and + minor version, e.g. 7.2.3 -> rocm72, 10.0.0 -> rocm100. + """ + return "rocm" + "".join(rocm_version.split(".")[:2]) @staticmethod def detect_system_rocm_variant() -> str | None: @@ -1382,7 +1384,9 @@ def get_vllm_version() -> str: # Get the Rocm Version rocm_version = get_rocm_version() or torch.version.hip if rocm_version and rocm_version != envs.VLLM_MAIN_CUDA_VERSION: - version += f"{sep}rocm{rocm_version.replace('.', '')[:3]}" + version += sep + precompiled_wheel_utils.rocm_version_to_variant( + rocm_version + ) elif _is_tpu(): version += f"{sep}tpu" elif _is_cpu(): @@ -1512,6 +1516,7 @@ def get_requirements() -> list[str]: requirements = _read_requirements("rocm.txt") if "develop" in sys.argv[1:] and "--no-deps" not in sys.argv[1:]: _check_requirements_preinstalled(requirements) + requirements += get_rocm_device_requirements() elif _is_tpu(): requirements = _read_requirements("tpu.txt") elif _is_cpu(): @@ -1523,6 +1528,24 @@ def get_requirements() -> list[str]: return requirements +def get_rocm_device_requirements() -> list[str]: + """GPU kernel packages for TheRock builds. + + TheRock ships GPU kernels as separate per-arch wheels selected through + ``[device-]`` extras on torch, torchvision and rocm. Require them for + every arch this wheel was compiled for, so ``pip install vllm`` works on + any supported GPU. + """ + if not _is_hip() or importlib.util.find_spec("rocm_sdk") is None: + return [] + arches = [a for a in os.getenv("PYTORCH_ROCM_ARCH", "").split(";") if a] + return [ + f"{pkg}[device-{arch}]" + for pkg in ("torch", "torchvision", "rocm") + for arch in arches + ] + + ext_modules = [] if _is_cuda() or _is_hip(): diff --git a/tests/models/language/pooling/test_token_classification.py b/tests/models/language/pooling/test_token_classification.py index c205a3c42a..9a8ac646a7 100644 --- a/tests/models/language/pooling/test_token_classification.py +++ b/tests/models/language/pooling/test_token_classification.py @@ -110,7 +110,7 @@ def test_modernbert_models( for hf_output, vllm_output in zip(hf_outputs, vllm_outputs): hf_output = hf_output.detach().clone().cpu().float() vllm_output = vllm_output.detach().clone().cpu().float() - torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3) + torch.testing.assert_close(hf_output, vllm_output, atol=6e-2, rtol=1e-3) @pytest.mark.parametrize("model", ["Davlan/xlm-roberta-base-ner-hrl"]) diff --git a/tools/vllm-rocm/generate-rocm-wheels-root-index.sh b/tools/vllm-rocm/generate-rocm-wheels-root-index.sh index 650a719378..8ccfab17f7 100755 --- a/tools/vllm-rocm/generate-rocm-wheels-root-index.sh +++ b/tools/vllm-rocm/generate-rocm-wheels-root-index.sh @@ -17,14 +17,20 @@ # # Environment variables: # S3_BUCKET - Bucket name (default: vllm-wheels) -# VARIANT - ROCm variant (default: rocm721) +# ROCM_BASE_DOCKERFILE - default ROCm stack whose variant the root index serves +# (default: docker/Dockerfile.rocm_base) +# VARIANT - override the ROCm variant derived from ROCM_BASE_DOCKERFILE # DRY_RUN - Set to 1 for preview mode (same as --dry-run) set -euo pipefail # ======== Configuration ======== BUCKET="${S3_BUCKET:-vllm-wheels}" -VARIANT="${VARIANT:-rocm721}" +if [[ -z "${VARIANT:-}" ]]; then + # shellcheck source=.buildkite/scripts/rocm/stack.sh + source .buildkite/scripts/rocm/stack.sh "${ROCM_BASE_DOCKERFILE:-docker/Dockerfile.rocm_base}" + VARIANT="$ROCM_STACK_VARIANT" +fi DRY_RUN="${DRY_RUN:-0}" FORCE_VERSION="" diff --git a/tools/vllm-rocm/therock_wheels.py b/tools/vllm-rocm/therock_wheels.py new file mode 100644 index 0000000000..3728ab4d08 --- /dev/null +++ b/tools/vllm-rocm/therock_wheels.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Adapt TheRock (ROCm 10) wheels for the vLLM ROCm wheel index. + +repack-torch: TheRock's torch wheel hard-pins the triton it was built with. + vLLM ships a newer source-built triton, so rewrite that one Requires-Dist + and the METADATA hash in RECORD. The version and filename stay the same, + because torchvision and the amd-torch-device-* wheels pin torch exactly. + +external-links: list the TheRock wheels the index links to instead of hosting + (ROCm SDK, device kernels, torchvision/torchaudio, rocm-bootstrap), with + versions and GPU arches read from docker/Dockerfile.rocm_base. torch's + device- extras can also pull in a GPU-family wheel (e.g. + amd-torch-device-gfx110x), so those are read from the torch wheel itself. + +patch-amdsmi: rocm-sdk-core ships the amdsmi bindings under + _rocm_sdk_core/share/amd_smi, and the loader finds libamd_smi relative to + that directory. Once installed as its own package it would miss the SDK + library, so also look it up through the installed _rocm_sdk_core package. +""" + +import argparse +import hashlib +import shutil +import sys +import tempfile +import urllib.request +import zipfile +from html.parser import HTMLParser +from pathlib import Path +from urllib.parse import unquote, urljoin + +import pybase64 as base64 +import regex as re + + +def _record_hash(data: bytes) -> str: + digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=") + return f"sha256={digest.decode()}" + + +def repack_torch(wheel: Path, triton_version: str, out_dir: Path) -> Path: + with zipfile.ZipFile(wheel) as src: + infos = src.infolist() + metadata = next(i for i in infos if i.filename.endswith(".dist-info/METADATA")) + record = next(i for i in infos if i.filename.endswith(".dist-info/RECORD")) + + text = src.read(metadata).decode() + new_text, count = re.subn( + r"^Requires-Dist: triton\s*==\s*\S+\s*$", + f"Requires-Dist: triton=={triton_version}", + text, + flags=re.MULTILINE, + ) + if count != 1: + raise SystemExit( + f"expected exactly one pinned triton requirement, found {count}" + ) + new_metadata = new_text.encode() + + lines = src.read(record).decode().splitlines() + prefix = metadata.filename + "," + replaced = [ + f"{metadata.filename},{_record_hash(new_metadata)},{len(new_metadata)}" + if line.startswith(prefix) + else line + for line in lines + ] + if replaced == lines: + raise SystemExit(f"{metadata.filename} not found in RECORD") + new_record = ("\n".join(replaced) + "\n").encode() + + out_dir.mkdir(parents=True, exist_ok=True) + out = out_dir / wheel.name + with zipfile.ZipFile(out, "w") as dst: + for info in infos: + if info.filename == metadata.filename: + data = new_metadata + elif info.filename == record.filename: + data = new_record + else: + data = src.read(info) + dst.writestr(info, data, compress_type=info.compress_type) + return out + + +_AMDSMI_ANCHOR = " possible_locations.append(libamd_smi_path)\n" +_AMDSMI_PATCH = """\ + # vLLM: installed as its own package, find the lib in TheRock's rocm-sdk-core. + try: + import importlib.util + _sdk_core = importlib.util.find_spec("_rocm_sdk_core") + if _sdk_core is not None and _sdk_core.origin: + possible_locations.append( + Path(_sdk_core.origin).parent / "lib/libamd_smi.so.27") + except Exception: + pass +""" + + +def patch_amdsmi(src_dir: Path, out_dir: Path) -> Path: + out_dir = out_dir.resolve() + if out_dir.exists(): + shutil.rmtree(out_dir) + shutil.copytree(src_dir, out_dir, ignore=shutil.ignore_patterns("tests")) + wrapper = out_dir / "amdsmi" / "amdsmi_wrapper.py" + text = wrapper.read_text() + if text.count(_AMDSMI_ANCHOR) != 1: + raise SystemExit(f"amdsmi loader anchor not found in {wrapper}") + wrapper.write_text(text.replace(_AMDSMI_ANCHOR, _AMDSMI_ANCHOR + _AMDSMI_PATCH)) + return out_dir + + +def dockerfile_args(dockerfile: Path) -> dict[str, str]: + args = {} + for m in re.finditer(r'^ARG (\w+)="?([^"\s]*)"?', dockerfile.read_text(), re.M): + args.setdefault(m.group(1), m.group(2)) + return args + + +class _Links(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.hrefs: list[str] = [] + + def handle_starttag(self, tag, attrs): + if tag == "a": + self.hrefs += [v for k, v in attrs if k == "href" and v] + + +def _project_files(index_url: str, project: str) -> list[str]: + page = urljoin(index_url, f"{project}/") + with urllib.request.urlopen(page, timeout=60) as resp: + parser = _Links() + parser.feed(resp.read().decode()) + return [urljoin(page, href.split("#")[0]) for href in parser.hrefs] + + +def _usable(url: str, version: str | None) -> bool: + name = unquote(url.rsplit("/", 1)[-1]) + if ( + version is not None + and f"-{version}-" not in name + and f"-{version}." not in name + ): + return False + if name.endswith(".tar.gz"): + return True + tags = name[: -len(".whl")].split("-")[-3:] + return ( + name.endswith(".whl") + and tags[0] in ("cp312", "py3") + and (tags[2] == "any" or ("x86_64" in tags[2] and "win" not in tags[2])) + ) + + +def torch_device_requirements( + torch_wheel: Path, arches: list[str] +) -> list[tuple[str, str]]: + """(project, version) pairs that torch's device- extras require.""" + with zipfile.ZipFile(torch_wheel) as zf: + metadata = next(n for n in zf.namelist() if n.endswith(".dist-info/METADATA")) + text = zf.read(metadata).decode() + wanted = {f"device-{arch}" for arch in arches} + reqs = [] + for m in re.finditer( + r'^Requires-Dist: ([\w.-]+)\s*==\s*(\S+)\s*;.*extra == "([^"]+)"', text, re.M + ): + if m.group(3) in wanted and (m.group(1), m.group(2)) not in reqs: + reqs.append((m.group(1), m.group(2))) + missing = wanted - set(re.findall(r'extra == "(device-[^"]+)"', text)) + if missing: + raise SystemExit(f"torch has no extras for {sorted(missing)}") + return reqs + + +def external_links(dockerfile: Path, torch_wheel: Path) -> list[str]: + args = dockerfile_args(dockerfile) + index = args["ROCM_RELEASE_WHEELS_MULTIARCH_URL"] + sdk = args["ROCM_SDK_VERSION"] + vision_v, audio_v = args["TORCHVISION_VERSION"], args["TORCHAUDIO_VERSION"] + arches = [a for a in args["PYTORCH_ROCM_ARCH"].split(";") if a] + wanted: list[tuple[str, str | None]] = [ + ("rocm", sdk), + ("rocm-sdk-core", sdk), + ("rocm-sdk-libraries", sdk), + ("torchvision", vision_v), + ("torchaudio", audio_v), + ("rocm-bootstrap", None), + ] + for arch in arches: + wanted += [ + (f"rocm-sdk-device-{arch}", sdk), + (f"amd-torchvision-device-{arch}", vision_v), + ] + wanted += torch_device_requirements(torch_wheel, arches) + urls = [] + for project, version in wanted: + found = [u for u in _project_files(index, project) if _usable(u, version)] + if not found: + raise SystemExit(f"no usable {project}=={version} files on {index}") + urls += found + return urls + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="cmd", required=True) + p = sub.add_parser("repack-torch") + p.add_argument("wheel", type=Path) + p.add_argument("--triton-version", required=True) + p.add_argument("--out-dir", type=Path, required=True) + p = sub.add_parser("external-links") + p.add_argument( + "--dockerfile", type=Path, default=Path("docker/Dockerfile.rocm_base") + ) + p.add_argument("--torch-wheel", type=Path, required=True) + p = sub.add_parser("patch-amdsmi") + p.add_argument("src_dir", type=Path) + p.add_argument("--out-dir", type=Path, default=None) + args = parser.parse_args() + + if args.cmd == "repack-torch": + print(repack_torch(args.wheel, args.triton_version, args.out_dir)) + elif args.cmd == "external-links": + print("\n".join(external_links(args.dockerfile, args.torch_wheel))) + else: + out = args.out_dir or Path(tempfile.mkdtemp()) / "amd_smi" + print(patch_amdsmi(args.src_dir, out)) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/vllm/platforms/rocm.py b/vllm/platforms/rocm.py index ec3d26c92c..4d0103f51d 100644 --- a/vllm/platforms/rocm.py +++ b/vllm/platforms/rocm.py @@ -405,6 +405,12 @@ if ( ): os.environ["HIP_ONLINE_TUNING"] = "1" +# RCCL 2.30's RAS thread can abort the process at exit ("terminate called +# without an active exception") when its shutdown notification fails. +# https://github.com/ROCm/rocm-systems/issues/13572 +if torch.version.hip: + os.environ.setdefault("NCCL_RAS_ENABLE", "0") + @cache def use_rocm_custom_paged_attention(