[Transformers backend] Enable QKV-Fuser for Gemma4 (#55690)
Signed-off-by: Thomas Ortner <boh@zurich.ibm.com> Signed-off-by: Thomas Ortner <bohnstinglthomas@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
08b3e67b66
commit
e509d32b5a
@@ -19,9 +19,8 @@ from vllm.model_executor.models.transformers.fx_utils import trace
|
||||
_FUSED_QKV_A_PROJ = MLAFuser.merged_name
|
||||
|
||||
|
||||
def _match(q_lora_rank: int | None) -> MLAFuser | None:
|
||||
"""Match a meta `DeepseekV2Attention` directly (bypassing the per-class
|
||||
`get_fuser` cache, so both q_lora variants of the same class are seen)."""
|
||||
def _attention(q_lora_rank: int | None):
|
||||
"""A meta `DeepseekV2Attention` in the requested q_lora configuration."""
|
||||
pytest.importorskip("transformers.models.deepseek_v2.modeling_deepseek_v2")
|
||||
from transformers.models.deepseek_v2.configuration_deepseek_v2 import (
|
||||
DeepseekV2Config,
|
||||
@@ -41,7 +40,13 @@ def _match(q_lora_rank: int | None) -> MLAFuser | None:
|
||||
num_hidden_layers=1,
|
||||
)
|
||||
with torch.device("meta"):
|
||||
attn = DeepseekV2Attention(cfg, layer_idx=0)
|
||||
return DeepseekV2Attention(cfg, layer_idx=0)
|
||||
|
||||
|
||||
def _match(q_lora_rank: int | None) -> MLAFuser | None:
|
||||
"""Match a meta `DeepseekV2Attention` directly (bypassing the per-class
|
||||
`get_fuser` cache, so both q_lora variants of the same class are seen)."""
|
||||
attn = _attention(q_lora_rank)
|
||||
return MLAFuser.match(trace(attn), attn)
|
||||
|
||||
|
||||
@@ -90,6 +95,23 @@ def test_q_lora_stacks_qkv_a_proj():
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("q_lora_rank", [64, None])
|
||||
def test_update_forward_rewrites_real_attention(q_lora_rank):
|
||||
"""The source rewrite must survive the input-stability check.
|
||||
|
||||
The q-LoRA path hoists the merged down-projection to a top-level statement
|
||||
while its calls sit inside the `q_lora_rank` branches, so the region it
|
||||
spans is wide; a check that over-approximates aliasing would silently refuse
|
||||
here and cost the fusion rather than fail loudly."""
|
||||
fuser = _match(q_lora_rank)
|
||||
assert isinstance(fuser, MLAFuser)
|
||||
fuser.update_forward(_attention(q_lora_rank))
|
||||
names = set(fuser.fused_forward.__code__.co_names)
|
||||
if fuser.has_q_lora:
|
||||
assert _FUSED_QKV_A_PROJ in names
|
||||
assert not {"q_a_proj", "kv_a_proj_with_mqa"} & names
|
||||
|
||||
|
||||
class _Norm(nn.Module):
|
||||
"""A real RMSNorm computation: `match` verifies chain norms via `RMSNormFuser`."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user