From 2c3fb4c545005db887788747b48d040d930604df Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Fri, 18 Sep 2026 16:13:35 -0500 Subject: [PATCH] [CI][ROCm] Deprecate DinD for MI250 test groups (#56162) Signed-off-by: Andreas Karatzas Signed-off-by: Andreas Karatzas Co-authored-by: OpenAI Codex --- .../scripts/hardware_ci/run-amd-test.sh | 8 ++++ .buildkite/test-amd.yaml | 22 ++++++++++- .buildkite/test_areas/cuda.yaml | 1 + .buildkite/test_areas/distributed.yaml | 1 + .buildkite/test_areas/docker.yaml | 1 + .buildkite/test_areas/engine.yaml | 3 ++ .buildkite/test_areas/kernels.yaml | 2 + .buildkite/test_areas/misc.yaml | 1 + .buildkite/test_areas/models_basic.yaml | 4 +- .buildkite/test_areas/models_language.yaml | 2 + .buildkite/test_areas/models_multimodal.yaml | 2 + .buildkite/test_areas/plugins.yaml | 1 + .../test_areas/rust_frontend_cargo.yaml | 2 + .buildkite/test_areas/samplers.yaml | 1 + tests/conftest.py | 2 +- .../model_executor/layers/test_activation.py | 38 +++++++++++++++++++ .../tools/test_docker_build_metadata_args.py | 7 +++- vllm/model_executor/layers/activation.py | 10 ++++- 18 files changed, 102 insertions(+), 6 deletions(-) create mode 100644 tests/model_executor/layers/test_activation.py diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index 541a8f5525..2705c3b87e 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -1612,6 +1612,13 @@ fi echo "Final commands: $commands" +# Match native CPU jobs even when the container can see AMD devices. +cpu_platform_env=() +if [[ "${VLLM_CI_EXPECTED_GPU_COUNT:-1}" == "0" \ + && "$commands" != *python_only_compile.sh* ]]; then + cpu_platform_env=(-e "VLLM_TARGET_DEVICE=cpu") +fi + standalone_merge_base_env=() if [[ "$commands" == *python_only_compile.sh* ]]; then # The ROCm test image often ships /vllm-workspace without .git. Resolve the @@ -1769,6 +1776,7 @@ else -e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \ -e "XDG_CACHE_HOME=${CONTAINER_CACHE_ROOT}/xdg" \ -e "PYTORCH_ROCM_ARCH=" \ + "${cpu_platform_env[@]}" \ "${standalone_merge_base_env[@]}" \ --name "${container_name}" \ "${image_name}" \ diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index ba8f382755..789ca54260 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -114,6 +114,7 @@ steps: - label: ":computer: (CPU) Basic Models Other" # TBD timeout_in_minutes: 35 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 no_gpu: true optional: true @@ -124,13 +125,15 @@ steps: - tests/models/test_vision.py - tests/models/test_adapters.py - tests/models/transformers/fusers/ + - tests/model_executor/layers/test_activation.py - tests/models/test_qwen3_5_mtp_config.py commands: - - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/test_qwen3_5_mtp_config.py models/transformers/fusers/ + - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/test_qwen3_5_mtp_config.py models/transformers/fusers/ model_executor/layers/test_activation.py - label: ":computer: (CPU) Multimodal Processor Shard %N" # TBD timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 no_gpu: true parallelism: 6 @@ -146,6 +149,7 @@ steps: - label: ":computer: (CPU) V1 Others" # TBD timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 no_gpu: true optional: true @@ -166,6 +170,7 @@ steps: - label: ":computer: (CPU) Async Engine, Inputs, Utils, Worker, Config" # TBD timeout_in_minutes: 80 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 no_gpu: true optional: true @@ -205,6 +210,7 @@ steps: - label: ":computer: (CPU) Rust Frontend Cargo Style + Clippy" # TBD timeout_in_minutes: 35 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 no_gpu: true optional: true @@ -220,6 +226,7 @@ steps: - label: ":computer: (CPU) Rust Frontend Cargo" # TBD timeout_in_minutes: 35 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 no_gpu: true optional: true @@ -235,6 +242,7 @@ steps: - label: ":computer: (CPU) Docker Build Metadata" # TBD timeout_in_minutes: 35 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 no_gpu: true optional: true @@ -265,6 +273,7 @@ steps: - label: ":amd: (MI250) PyTorch Fullgraph Smoke" # TBD timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -284,6 +293,7 @@ steps: - label: ":amd: (MI250) Pipeline + Context Parallelism" # TBD timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_4 num_gpus: 4 working_dir: "/vllm-workspace/tests" @@ -311,6 +321,7 @@ steps: timeout_in_minutes: 45 parallelism: 2 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -328,6 +339,7 @@ steps: - label: ":amd: (MI250) Language Models (PPL)" # TBD timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -359,6 +371,7 @@ steps: - label: ":amd: (MI250) Multimodal Models (Standard) 3: llava + qwen2_vl" # TBD timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -374,6 +387,7 @@ steps: - label: ":amd: (MI250) Batch Invariance" # TBD timeout_in_minutes: 35 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -392,6 +406,7 @@ steps: - label: ":amd: (MI250) CUDAGraph" # TBD timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -412,6 +427,7 @@ steps: - label: ":amd: (MI250) E2E Core" # TBD timeout_in_minutes: 50 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -425,6 +441,7 @@ steps: - label: ":amd: (MI250) E2E Scheduling" # TBD timeout_in_minutes: 55 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -438,6 +455,7 @@ steps: - label: ":amd: (MI250) V1 Engine" # TBD timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -460,6 +478,7 @@ steps: - label: ":amd: (MI250) Spec Decode Draft Model" # TBD timeout_in_minutes: 60 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" @@ -477,6 +496,7 @@ steps: - label: ":amd: (MI250) Spec Decode Speculators + MTP" # TBD timeout_in_minutes: 75 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + dind: false agent_pool: mi250_1 optional: true working_dir: "/vllm-workspace/tests" diff --git a/.buildkite/test_areas/cuda.yaml b/.buildkite/test_areas/cuda.yaml index a031c37ba3..8ac8e1960a 100644 --- a/.buildkite/test_areas/cuda.yaml +++ b/.buildkite/test_areas/cuda.yaml @@ -49,6 +49,7 @@ steps: mirror: amd: label: ":amd: (MI250) CUDAGraph" + dind: false device: mi250_1 timeout_in_minutes: 55 depends_on: diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index b83d2893bd..a108246563 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -416,6 +416,7 @@ steps: mirror: amd: label: ":amd: (MI250) Pipeline + Context Parallelism" + dind: false device: mi250_4 timeout_in_minutes: 90 depends_on: diff --git a/.buildkite/test_areas/docker.yaml b/.buildkite/test_areas/docker.yaml index ef50062a7f..4c80f26ca5 100644 --- a/.buildkite/test_areas/docker.yaml +++ b/.buildkite/test_areas/docker.yaml @@ -18,6 +18,7 @@ steps: mirror: amd: label: ":amd: (MI250) Docker Build Metadata" + dind: false device: mi250_1 no_gpu: true timeout_in_minutes: 35 diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index f6e243a1b7..8e47a84e18 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -52,6 +52,7 @@ steps: mirror: amd: label: ":amd: (MI250) V1 Engine" + dind: false device: mi250_1 timeout_in_minutes: 45 depends_on: @@ -71,6 +72,7 @@ steps: mirror: amd: label: ":amd: (MI250) E2E Scheduling" + dind: false device: mi250_1 timeout_in_minutes: 55 depends_on: @@ -90,6 +92,7 @@ steps: mirror: amd: label: ":amd: (MI250) E2E Core" + dind: false commands: - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py device: mi250_1 diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index 207d827f36..7e43b123f1 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -322,6 +322,7 @@ steps: mirror: amd: label: ":amd: (MI250) Mamba Kernels" + dind: false device: mi250_1 timeout_in_minutes: 55 depends_on: @@ -512,6 +513,7 @@ steps: mirror: amd: label: ":amd: (MI250) Helion Kernels Shard %N" + dind: false device: mi250_1 timeout_in_minutes: 45 depends_on: diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index f17d154fbe..1bff8e1850 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -586,6 +586,7 @@ steps: mirror: amd: label: ":amd: (MI250) Batch Invariance" + dind: false device: mi250_1 timeout_in_minutes: 35 depends_on: diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index 94b0723f8a..6702eb6dd3 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -170,12 +170,14 @@ steps: - tests/models/test_adapters.py - tests/models/test_qwen3_5_mtp_config.py - tests/models/transformers/fusers/ + - tests/model_executor/layers/test_activation.py device: cpu-small commands: - - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/test_qwen3_5_mtp_config.py models/transformers/fusers/ + - pytest -v -s models/test_utils.py models/test_vision.py models/test_adapters.py models/test_qwen3_5_mtp_config.py models/transformers/fusers/ model_executor/layers/test_activation.py mirror: amd: label: ":amd: (MI250) Basic Models Other" + dind: false device: mi250_1 no_gpu: true working_dir: "/vllm-workspace/tests" diff --git a/.buildkite/test_areas/models_language.yaml b/.buildkite/test_areas/models_language.yaml index 3e5024fe87..3f80dfdfa3 100644 --- a/.buildkite/test_areas/models_language.yaml +++ b/.buildkite/test_areas/models_language.yaml @@ -155,6 +155,7 @@ steps: mirror: amd: label: ":amd: (MI250) Language Models (PPL)" + dind: false device: mi250_1 timeout_in_minutes: 65 depends_on: @@ -214,6 +215,7 @@ steps: mirror: amd: label: ":amd: (MI250) Language Models (MTEB)" + dind: false device: mi250_1 timeout_in_minutes: 60 depends_on: diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml index a324da687f..779801a2ea 100644 --- a/.buildkite/test_areas/models_multimodal.yaml +++ b/.buildkite/test_areas/models_multimodal.yaml @@ -57,6 +57,7 @@ steps: mirror: amd: label: ":amd: (MI250) Multimodal Models (Standard) 3: llava + qwen2_vl" + dind: false device: mi250_1 timeout_in_minutes: 55 depends_on: @@ -101,6 +102,7 @@ steps: mirror: amd: label: ":amd: (MI250) Multimodal Processor Shard %N" + dind: false device: mi250_1 no_gpu: true working_dir: "/vllm-workspace/tests" diff --git a/.buildkite/test_areas/plugins.yaml b/.buildkite/test_areas/plugins.yaml index 11f63ff52d..6b901d8f49 100644 --- a/.buildkite/test_areas/plugins.yaml +++ b/.buildkite/test_areas/plugins.yaml @@ -53,6 +53,7 @@ steps: mirror: amd: label: ":amd: (MI250) Plugin Integration" + dind: false device: mi250_2 timeout_in_minutes: 50 depends_on: diff --git a/.buildkite/test_areas/rust_frontend_cargo.yaml b/.buildkite/test_areas/rust_frontend_cargo.yaml index 0b49787d5d..53d1da7edf 100644 --- a/.buildkite/test_areas/rust_frontend_cargo.yaml +++ b/.buildkite/test_areas/rust_frontend_cargo.yaml @@ -17,6 +17,7 @@ steps: mirror: amd: label: ":amd: (MI250) Rust Frontend Cargo Style + Clippy" + dind: false device: mi250_1 no_gpu: true working_dir: "/vllm-workspace" @@ -40,6 +41,7 @@ steps: mirror: amd: label: ":amd: (MI250) Rust Frontend Cargo" + dind: false device: mi250_1 no_gpu: true working_dir: "/vllm-workspace" diff --git a/.buildkite/test_areas/samplers.yaml b/.buildkite/test_areas/samplers.yaml index 18f857f65a..e1bc4c08d8 100644 --- a/.buildkite/test_areas/samplers.yaml +++ b/.buildkite/test_areas/samplers.yaml @@ -20,6 +20,7 @@ steps: mirror: amd: label: ":amd: (MI250) Samplers" + dind: false device: mi250_1 timeout_in_minutes: 40 depends_on: diff --git a/tests/conftest.py b/tests/conftest.py index a52a38f357..0d70be62eb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -241,7 +241,7 @@ def dist_init(): rank=0, distributed_init_method=f"file://{temp_file}", local_rank=0, - backend="nccl", + backend="gloo" if current_platform.is_cpu() else "nccl", ) initialize_model_parallel(1, 1) yield diff --git a/tests/model_executor/layers/test_activation.py b/tests/model_executor/layers/test_activation.py new file mode 100644 index 0000000000..d0ef42c99a --- /dev/null +++ b/tests/model_executor/layers/test_activation.py @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from functools import partial + +import pytest +import torch +import torch.nn as nn + +from vllm.model_executor.layers.activation import GeluAndMul, SiluAndMul +from vllm.platforms import CpuArchEnum, current_platform + + +@pytest.mark.skipif( + not current_platform.is_cpu() + or current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC, + reason="Native CPU activations require a non-PowerPC CPU platform", +) +@pytest.mark.parametrize( + "activation_factory,reference", + [ + (SiluAndMul, nn.SiLU()), + (GeluAndMul, nn.GELU()), + (partial(GeluAndMul, approximate="tanh"), nn.GELU(approximate="tanh")), + ], + ids=["silu", "gelu", "gelu_tanh"], +) +def test_cpu_act_and_mul_without_compiled_ops( + default_vllm_config, monkeypatch, activation_factory, reference +): + """CPU activations must construct and run without compiled wheel operators.""" + default_vllm_config.compilation_config.custom_ops = ["all"] + monkeypatch.setattr(torch.ops, "_C", object()) + x = torch.tensor([[-2.0, -1.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0]], device="cpu") + + layer = activation_factory() + + torch.testing.assert_close(layer(x), reference(x[:, :4]) * x[:, 4:]) diff --git a/tests/tools/test_docker_build_metadata_args.py b/tests/tools/test_docker_build_metadata_args.py index 8095ad34aa..127768afc7 100644 --- a/tests/tools/test_docker_build_metadata_args.py +++ b/tests/tools/test_docker_build_metadata_args.py @@ -155,10 +155,15 @@ def test_vllm_openai_image_embeds_metadata_contract() -> None: def test_rust_build_cache_excludes_git_metadata() -> None: + import torch + from vllm.platforms import current_platform dockerfile_names = ["Dockerfile", "Dockerfile.cpu"] - if not current_platform.is_rocm(): + # CPU jobs can reuse ROCm artifacts, which omit the XPU Dockerfile. + if not current_platform.is_rocm() and ( + not current_platform.is_cpu() or torch.version.hip is None + ): dockerfile_names.append("Dockerfile.xpu") for name in dockerfile_names: dockerfile = (REPO_ROOT / "docker" / name).read_text() diff --git a/vllm/model_executor/layers/activation.py b/vllm/model_executor/layers/activation.py index 7166d3f8fe..35a6fdf9b0 100644 --- a/vllm/model_executor/layers/activation.py +++ b/vllm/model_executor/layers/activation.py @@ -129,8 +129,11 @@ class SiluAndMul(CustomOp): super().__init__(compile_native=compile_native) if ( current_platform.is_cuda_alike() - or current_platform.is_cpu() or current_platform.is_xpu() + or ( + current_platform.is_cpu() + and current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC + ) ): self.op = torch.ops._C.silu_and_mul @@ -424,8 +427,11 @@ class GeluAndMul(CustomOp): raise ValueError(f"Unknown approximate mode: {approximate}") if ( current_platform.is_cuda_alike() - or current_platform.is_cpu() or current_platform.is_xpu() + or ( + current_platform.is_cpu() + and current_platform.get_cpu_architecture() == CpuArchEnum.POWERPC + ) ): if approximate == "none": self.op = torch.ops._C.gelu_and_mul