[Kernel][Perf] Support Qwen head ratios in fused GDN MTP (#52539)
Signed-off-by: kiroxu <148877251+BabyDrangoner@users.noreply.github.com>
This commit is contained in:
@@ -5,6 +5,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import types
|
||||
from typing import cast
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
@@ -37,6 +38,7 @@ from vllm.third_party.flash_linear_attention.ops.layernorm_guard import ( # noq
|
||||
)
|
||||
from vllm.utils.torch_utils import _encode_layer_name # noqa: E402
|
||||
from vllm.v1.attention.backends.gdn_attn import ( # noqa: E402
|
||||
GDNAttentionMetadata,
|
||||
GDNAttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import MambaSpec # noqa: E402
|
||||
@@ -139,6 +141,46 @@ def _build_layer(
|
||||
return layer
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_v_heads,expected",
|
||||
[
|
||||
pytest.param(2, True, id="ratio1"),
|
||||
pytest.param(4, True, id="ratio2"),
|
||||
pytest.param(6, True, id="ratio3"),
|
||||
pytest.param(8, True, id="ratio4"),
|
||||
pytest.param(16, True, id="ratio8"),
|
||||
pytest.param(5, False, id="non-integral-ratio"),
|
||||
pytest.param(10, False, id="unsupported-ratio5"),
|
||||
],
|
||||
)
|
||||
def test_fused_mtp_head_ratio_guard(num_v_heads: int, expected: bool) -> None:
|
||||
if not hasattr(torch.ops._C, "fused_gdn_decode_post_conv_mtp"):
|
||||
pytest.skip("fused GDN decode MTP op is not built")
|
||||
|
||||
layer = types.SimpleNamespace(
|
||||
num_k_heads=2,
|
||||
num_v_heads=num_v_heads,
|
||||
kv_cache=(None, torch.empty(1, dtype=torch.float32, device="cuda")),
|
||||
gdn_decode_kernel="cuda",
|
||||
)
|
||||
attn_metadata = types.SimpleNamespace(
|
||||
spec_state_indices_tensor=torch.ones(
|
||||
1, SPEC_TOKENS, dtype=torch.int32, device="cuda"
|
||||
),
|
||||
spec_sequence_masks=object(),
|
||||
num_decodes=0,
|
||||
num_spec_decodes=1,
|
||||
)
|
||||
|
||||
assert (
|
||||
QwenGatedDeltaNetAttention._can_use_fused_gdn_mtp_decode(
|
||||
cast(QwenGatedDeltaNetAttention, layer),
|
||||
cast(GDNAttentionMetadata, attn_metadata),
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_fused_forward_uses_packed_entrypoint() -> None:
|
||||
"""Fused mode keeps projected QKVZ and BA packed through the model op."""
|
||||
|
||||
@@ -217,18 +217,59 @@ def test_fused_post_conv_l0():
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"tp_size,query_lengths,state_dtype,norm_dtype",
|
||||
"head_ratio,tp_size,query_lengths,state_dtype,norm_dtype",
|
||||
[
|
||||
pytest.param(16, (4, 4), torch.bfloat16, torch.bfloat16, id="tp16-bf16"),
|
||||
pytest.param(4, (4, 4), torch.float32, torch.float32, id="tp4-fp32"),
|
||||
pytest.param(16, (4, 2, 0), torch.bfloat16, torch.float32, id="tp16-ragged"),
|
||||
pytest.param(4, (4, 2, 0), torch.float32, torch.bfloat16, id="tp4-ragged"),
|
||||
pytest.param(16, (8,), torch.float32, torch.bfloat16, id="tp16-max"),
|
||||
pytest.param(4, (8,), torch.bfloat16, torch.float32, id="tp4-max"),
|
||||
pytest.param(8, 16, (4, 4), torch.bfloat16, torch.bfloat16, id="tp16-bf16"),
|
||||
pytest.param(8, 4, (4, 4), torch.float32, torch.float32, id="tp4-fp32"),
|
||||
pytest.param(8, 16, (4, 2, 0), torch.bfloat16, torch.float32, id="tp16-ragged"),
|
||||
pytest.param(8, 4, (4, 2, 0), torch.float32, torch.bfloat16, id="tp4-ragged"),
|
||||
pytest.param(8, 16, (8,), torch.float32, torch.bfloat16, id="tp16-max"),
|
||||
pytest.param(8, 4, (8,), torch.bfloat16, torch.float32, id="tp4-max"),
|
||||
pytest.param(
|
||||
1,
|
||||
1,
|
||||
(4, 4),
|
||||
torch.float32,
|
||||
torch.bfloat16,
|
||||
id="ratio1-tp1-fp32",
|
||||
),
|
||||
pytest.param(
|
||||
2,
|
||||
1,
|
||||
(4, 2, 0),
|
||||
torch.bfloat16,
|
||||
torch.bfloat16,
|
||||
id="ratio2-tp1-ragged-bf16",
|
||||
),
|
||||
pytest.param(
|
||||
2,
|
||||
4,
|
||||
(8,),
|
||||
torch.float32,
|
||||
torch.float32,
|
||||
id="ratio2-tp4-max-fp32",
|
||||
),
|
||||
pytest.param(
|
||||
3,
|
||||
2,
|
||||
(4, 2, 0),
|
||||
torch.float32,
|
||||
torch.float32,
|
||||
id="ratio3-tp2-ragged-fp32",
|
||||
),
|
||||
pytest.param(
|
||||
4,
|
||||
4,
|
||||
(8,),
|
||||
torch.float32,
|
||||
torch.bfloat16,
|
||||
id="ratio4-tp4-max-fp32",
|
||||
),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_fused_gdn_decode_post_conv_mtp_ratio8(
|
||||
def test_fused_gdn_decode_post_conv_mtp_head_ratios(
|
||||
head_ratio: int,
|
||||
tp_size: int,
|
||||
query_lengths: tuple[int, ...],
|
||||
state_dtype: torch.dtype,
|
||||
@@ -242,7 +283,7 @@ def test_fused_gdn_decode_post_conv_mtp_ratio8(
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
H = 16 // tp_size
|
||||
HV = 128 // tp_size
|
||||
HV = head_ratio * H
|
||||
K = V = 128
|
||||
num_reqs = len(query_lengths)
|
||||
state_width = max(query_lengths)
|
||||
|
||||
Reference in New Issue
Block a user