PD disagg with NIXL Connector: GDN support (Qwen3.5) (#41869)
Signed-off-by: Zhanqiu Hu <zhu@redhat.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
parent
c7560af424
commit
24337fb860
@@ -21,6 +21,9 @@ hybrid_ssm_configs=(
|
||||
"VLLM_SSM_CONV_STATE_LAYOUT=DS ENABLE_HMA_FLAG=1 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=ibm-granite/granite-4.0-h-tiny VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192,--trust-remote-code"
|
||||
# TODO: (NickLucche) Address async scheduling issue with TP>1 separately as this may impact other models.
|
||||
"VLLM_SSM_CONV_STATE_LAYOUT=DS ENABLE_HMA_FLAG=1 PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=2 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=ibm-granite/granite-4.0-h-tiny VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192,--trust-remote-code,--no-async-scheduling"
|
||||
# GDN (Qwen3.5)
|
||||
"VLLM_SSM_CONV_STATE_LAYOUT=DS ENABLE_HMA_FLAG=1 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B"
|
||||
"VLLM_SSM_CONV_STATE_LAYOUT=DS ENABLE_HMA_FLAG=1 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B VLLM_SERVE_EXTRA_ARGS=--no-async-scheduling"
|
||||
)
|
||||
sw_attn_configs=(
|
||||
# NOTE: gemma3 does not work with FlashInfer
|
||||
|
||||
@@ -20,6 +20,7 @@ EXPECTED_VALUES = {
|
||||
"google/gemma-3-4b-it": 0.74,
|
||||
"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8": 0.84,
|
||||
"ibm-granite/granite-4.0-h-tiny": 0.80,
|
||||
"Qwen/Qwen3.5-0.8B": 0.33,
|
||||
}
|
||||
|
||||
SIMPLE_PROMPT = (
|
||||
|
||||
@@ -742,6 +742,132 @@ def test_compute_physical_blocks_per_logical(ssm_sizes, block_len, expected_rati
|
||||
assert compute_physical_blocks_per_logical(ssm_sizes, block_len) == expected_ratio
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
"mamba_type,local_tp,conv_dim_local,conv_rows,temporal_shape,expected_proj_dims",
|
||||
[
|
||||
# nvidia/Nemotron-H-8B-Base-8K (Mamba2)
|
||||
# mamba_num_heads=128, head_dim=64, n_groups=8, ssm_state_size=128
|
||||
pytest.param(
|
||||
"mamba2",
|
||||
1,
|
||||
10240,
|
||||
3,
|
||||
(128, 64, 128),
|
||||
(8192, 1024, 1024),
|
||||
id="nemotron_h_8b_tp1",
|
||||
),
|
||||
pytest.param(
|
||||
"mamba2",
|
||||
4,
|
||||
2560,
|
||||
3,
|
||||
(32, 64, 128),
|
||||
(2048, 256, 256),
|
||||
id="nemotron_h_8b_tp4",
|
||||
),
|
||||
# nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B (Mamba2)
|
||||
# mamba_num_heads=64, head_dim=64, n_groups=8, ssm_state_size=128
|
||||
pytest.param(
|
||||
"mamba2",
|
||||
1,
|
||||
6144,
|
||||
3,
|
||||
(64, 64, 128),
|
||||
(4096, 1024, 1024),
|
||||
id="nemotron_nano_30b_tp1",
|
||||
),
|
||||
# Qwen/Qwen3.5-0.8B (GDN, symmetric: num_v=num_k=16)
|
||||
# key_dim=2048, value_dim=2048, conv_dim=6144
|
||||
pytest.param(
|
||||
"gdn_attention",
|
||||
1,
|
||||
6144,
|
||||
3,
|
||||
(16, 128, 128),
|
||||
(2048, 2048, 2048),
|
||||
id="qwen35_08b_tp1",
|
||||
),
|
||||
pytest.param(
|
||||
"gdn_attention",
|
||||
4,
|
||||
1536,
|
||||
3,
|
||||
(4, 128, 128),
|
||||
(512, 512, 512),
|
||||
id="qwen35_08b_tp4",
|
||||
),
|
||||
# Qwen/Qwen3.5-4B (GDN, asymmetric: num_v=32, num_k=16, K:V=1:2)
|
||||
# key_dim=2048, value_dim=4096, conv_dim=8192
|
||||
pytest.param(
|
||||
"gdn_attention",
|
||||
1,
|
||||
8192,
|
||||
3,
|
||||
(32, 128, 128),
|
||||
(2048, 2048, 4096),
|
||||
id="qwen35_4b_tp1",
|
||||
),
|
||||
# Qwen/Qwen3.5-27B (GDN, asymmetric: num_v=48, num_k=16, K:V=1:3)
|
||||
# key_dim=2048, value_dim=6144, conv_dim=10240
|
||||
pytest.param(
|
||||
"gdn_attention",
|
||||
1,
|
||||
10240,
|
||||
3,
|
||||
(48, 128, 128),
|
||||
(2048, 2048, 6144),
|
||||
id="qwen35_27b_tp1",
|
||||
),
|
||||
pytest.param(
|
||||
"gdn_attention",
|
||||
8,
|
||||
1280,
|
||||
3,
|
||||
(6, 128, 128),
|
||||
(256, 256, 768),
|
||||
id="qwen35_27b_tp8",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_derive_mamba_conv_split(
|
||||
monkeypatch,
|
||||
mamba_type,
|
||||
local_tp,
|
||||
conv_dim_local,
|
||||
conv_rows,
|
||||
temporal_shape,
|
||||
expected_proj_dims,
|
||||
):
|
||||
"""Parametrized test for derive_mamba_conv_split with real model configs.
|
||||
|
||||
Values generated by verify_conv_split.py which loads HuggingFace configs
|
||||
and calls vLLM's derive_mamba_conv_split directly.
|
||||
"""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import (
|
||||
derive_mamba_conv_split,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
_TYPE_MAP = {
|
||||
"mamba2": MambaAttentionBackendEnum.MAMBA2,
|
||||
"gdn_attention": MambaAttentionBackendEnum.GDN_ATTN,
|
||||
}
|
||||
mamba_type_enum = _TYPE_MAP[mamba_type]
|
||||
|
||||
monkeypatch.setenv("VLLM_SSM_CONV_STATE_LAYOUT", "DS")
|
||||
spec = MambaSpec(
|
||||
block_size=64,
|
||||
shapes=((conv_dim_local, conv_rows), temporal_shape),
|
||||
dtypes=(torch.bfloat16, torch.bfloat16),
|
||||
mamba_type=mamba_type_enum,
|
||||
)
|
||||
out = derive_mamba_conv_split(spec, local_tp=local_tp)
|
||||
assert out.local_proj_dims == expected_proj_dims
|
||||
assert out.conv_rows == conv_rows
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
"mamba_enabled,swa_enabled,"
|
||||
|
||||
Reference in New Issue
Block a user