From f395a6b1ad49458ea5b0366805c3814c51d6b02b Mon Sep 17 00:00:00 2001 From: Rishabh Agarwal Date: Tue, 13 Oct 2026 11:03:48 -0400 Subject: [PATCH] [ROCM][Bugfix] Deferring to TRITON_ATTN backend for Prefix lm models instead of ROCM_ATTN and fixed minor bug in Gemma3 Multimodal code (#46754) Signed-off-by: GodRishUniverse Signed-off-by: Rishabh Agarwal Signed-off-by: GodRishUniverse --- .../test_rocm_attention_backends_selection.py | 25 +++++++++++++++++++ vllm/model_executor/models/gemma3_mm.py | 4 +++ 2 files changed, 29 insertions(+) diff --git a/tests/v1/attention/test_rocm_attention_backends_selection.py b/tests/v1/attention/test_rocm_attention_backends_selection.py index e0247a94a1..45580b395f 100644 --- a/tests/v1/attention/test_rocm_attention_backends_selection.py +++ b/tests/v1/attention/test_rocm_attention_backends_selection.py @@ -61,6 +61,31 @@ def test_aiter_unified_attention_uses_dedicated_metadata_builder(): ) +def test_rocm_backends_do_not_support_mm_prefix(): + """Regression test: ROCm backends must not claim mm_prefix support. + + If they do, prefix-LM multimodal models that attend vision tokens + bidirectionally (e.g. Gemma3 4B) will use bidirectional attention + via the ROCm backend instead of falling back to Triton, producing + incorrect results. Remove this test only if support is properly added. + """ + from vllm.v1.attention.backends.rocm_aiter_unified_attn import ( + RocmAiterUnifiedAttentionBackend, + ) + from vllm.v1.attention.backends.rocm_attn import RocmAttentionBackend + + assert not RocmAttentionBackend.supports_mm_prefix(), ( + "RocmAttentionBackend does not support mm_prefix — " + "defer to Triton unified attention instead for now" + "See https://github.com/vllm-project/vllm/issues/48172" + ) + assert not RocmAiterUnifiedAttentionBackend.supports_mm_prefix(), ( + "RocmAiterUnifiedAttentionBackend does not support mm_prefix — " + "defer to Triton unified attention instead for now" + "See https://github.com/ROCm/aiter/issues/4168" + ) + + def test_segmented_attention_uses_dedicated_backend_components(): from vllm.v1.attention.backends.rocm_segmented_attn import ( RocmSegmentedAttentionBackend, diff --git a/vllm/model_executor/models/gemma3_mm.py b/vllm/model_executor/models/gemma3_mm.py index c6166bc2f4..b5b0c0d48b 100644 --- a/vllm/model_executor/models/gemma3_mm.py +++ b/vllm/model_executor/models/gemma3_mm.py @@ -557,6 +557,10 @@ class Gemma3ForConditionalGeneration( ) self.multi_modal_projector = Gemma3MultiModalProjector(config) + # https://huggingface.co/blog/gemma3, (image tokens attend to each other) + # explicit mention here as text side in gemma3 standalone does not need + # prefix_lm but multimodal does + config.text_config.is_mm_prefix_lm = True with self._mark_language_model(vllm_config): self.language_model = init_vllm_registered_model( vllm_config=vllm_config,