[Transformers backend] Enable QKV-Fuser for Gemma4 (#55690)

Signed-off-by: Thomas Ortner <boh@zurich.ibm.com>
Signed-off-by: Thomas Ortner <bohnstinglthomas@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Thomas Ortner
2026-09-09 14:46:31 +01:00
committed by GitHub
co-authored by Harry Mellor
parent 08b3e67b66
commit e509d32b5a
8 changed files with 1035 additions and 90 deletions
+26 -4
View File
@@ -19,9 +19,8 @@ from vllm.model_executor.models.transformers.fx_utils import trace
_FUSED_QKV_A_PROJ = MLAFuser.merged_name
def _match(q_lora_rank: int | None) -> MLAFuser | None:
"""Match a meta `DeepseekV2Attention` directly (bypassing the per-class
`get_fuser` cache, so both q_lora variants of the same class are seen)."""
def _attention(q_lora_rank: int | None):
"""A meta `DeepseekV2Attention` in the requested q_lora configuration."""
pytest.importorskip("transformers.models.deepseek_v2.modeling_deepseek_v2")
from transformers.models.deepseek_v2.configuration_deepseek_v2 import (
DeepseekV2Config,
@@ -41,7 +40,13 @@ def _match(q_lora_rank: int | None) -> MLAFuser | None:
num_hidden_layers=1,
)
with torch.device("meta"):
attn = DeepseekV2Attention(cfg, layer_idx=0)
return DeepseekV2Attention(cfg, layer_idx=0)
def _match(q_lora_rank: int | None) -> MLAFuser | None:
"""Match a meta `DeepseekV2Attention` directly (bypassing the per-class
`get_fuser` cache, so both q_lora variants of the same class are seen)."""
attn = _attention(q_lora_rank)
return MLAFuser.match(trace(attn), attn)
@@ -90,6 +95,23 @@ def test_q_lora_stacks_qkv_a_proj():
}
@pytest.mark.parametrize("q_lora_rank", [64, None])
def test_update_forward_rewrites_real_attention(q_lora_rank):
"""The source rewrite must survive the input-stability check.
The q-LoRA path hoists the merged down-projection to a top-level statement
while its calls sit inside the `q_lora_rank` branches, so the region it
spans is wide; a check that over-approximates aliasing would silently refuse
here and cost the fusion rather than fail loudly."""
fuser = _match(q_lora_rank)
assert isinstance(fuser, MLAFuser)
fuser.update_forward(_attention(q_lora_rank))
names = set(fuser.fused_forward.__code__.co_names)
if fuser.has_q_lora:
assert _FUSED_QKV_A_PROJ in names
assert not {"q_a_proj", "kv_a_proj_with_mqa"} & names
class _Norm(nn.Module):
"""A real RMSNorm computation: `match` verifies chain norms via `RMSNormFuser`."""