From 8fd69754798c50b7b07938451bd97b1e66765927 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Wed, 25 Feb 2026 10:48:37 -0600 Subject: [PATCH] [ROCm][CI] Disable skinny GEMMs in multimodal tests to fix non-deterministic results (#35049) Signed-off-by: Andreas Karatzas --- tests/models/multimodal/conftest.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/models/multimodal/conftest.py b/tests/models/multimodal/conftest.py index 3f53b3fe62..d00c3df786 100644 --- a/tests/models/multimodal/conftest.py +++ b/tests/models/multimodal/conftest.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Pytest configuration for vLLM multimodal tests.""" +import os import warnings import torch @@ -9,6 +10,23 @@ import torch from vllm.platforms import current_platform +def pytest_configure(config): + """Early ROCm configuration that must happen before test collection.""" + if not current_platform.is_rocm(): + return + + # Disable skinny GEMM on ROCm to avoid non-deterministic results + # from atomic reductions in wvSplitKrc kernel. + # See: https://github.com/vllm-project/vllm/pull/33493#issuecomment-3906083975 + os.environ["VLLM_ROCM_USE_SKINNY_GEMM"] = "0" + warnings.warn( + "ROCm: Set VLLM_ROCM_USE_SKINNY_GEMM=0 to avoid non-deterministic " + "results from skinny GEMM atomic reductions", + UserWarning, + stacklevel=1, + ) + + def pytest_collection_modifyitems(config, items): """Configure ROCm-specific settings based on collected tests.""" if not current_platform.is_rocm():