PD disagg with NIXL Connector: GDN support (Qwen3.5) (#41869)

Signed-off-by: Zhanqiu Hu <zhu@redhat.com>
Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
zhanqiuhu
2026-05-14 16:33:01 +02:00
committed by GitHub
co-authored by Nicolò Lucchesi
parent c7560af424
commit 24337fb860
6 changed files with 244 additions and 83 deletions
@@ -21,6 +21,9 @@ hybrid_ssm_configs=(
"VLLM_SSM_CONV_STATE_LAYOUT=DS ENABLE_HMA_FLAG=1 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=ibm-granite/granite-4.0-h-tiny VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192,--trust-remote-code"
# TODO: (NickLucche) Address async scheduling issue with TP>1 separately as this may impact other models.
"VLLM_SSM_CONV_STATE_LAYOUT=DS ENABLE_HMA_FLAG=1 PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=2 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=ibm-granite/granite-4.0-h-tiny VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192,--trust-remote-code,--no-async-scheduling"
# GDN (Qwen3.5)
"VLLM_SSM_CONV_STATE_LAYOUT=DS ENABLE_HMA_FLAG=1 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B"
"VLLM_SSM_CONV_STATE_LAYOUT=DS ENABLE_HMA_FLAG=1 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B VLLM_SERVE_EXTRA_ARGS=--no-async-scheduling"
)
sw_attn_configs=(
# NOTE: gemma3 does not work with FlashInfer
@@ -20,6 +20,7 @@ EXPECTED_VALUES = {
"google/gemma-3-4b-it": 0.74,
"nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8": 0.84,
"ibm-granite/granite-4.0-h-tiny": 0.80,
"Qwen/Qwen3.5-0.8B": 0.33,
}
SIMPLE_PROMPT = (
@@ -742,6 +742,132 @@ def test_compute_physical_blocks_per_logical(ssm_sizes, block_len, expected_rati
assert compute_physical_blocks_per_logical(ssm_sizes, block_len) == expected_ratio
@pytest.mark.cpu_test
@pytest.mark.parametrize(
"mamba_type,local_tp,conv_dim_local,conv_rows,temporal_shape,expected_proj_dims",
[
# nvidia/Nemotron-H-8B-Base-8K (Mamba2)
# mamba_num_heads=128, head_dim=64, n_groups=8, ssm_state_size=128
pytest.param(
"mamba2",
1,
10240,
3,
(128, 64, 128),
(8192, 1024, 1024),
id="nemotron_h_8b_tp1",
),
pytest.param(
"mamba2",
4,
2560,
3,
(32, 64, 128),
(2048, 256, 256),
id="nemotron_h_8b_tp4",
),
# nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B (Mamba2)
# mamba_num_heads=64, head_dim=64, n_groups=8, ssm_state_size=128
pytest.param(
"mamba2",
1,
6144,
3,
(64, 64, 128),
(4096, 1024, 1024),
id="nemotron_nano_30b_tp1",
),
# Qwen/Qwen3.5-0.8B (GDN, symmetric: num_v=num_k=16)
# key_dim=2048, value_dim=2048, conv_dim=6144
pytest.param(
"gdn_attention",
1,
6144,
3,
(16, 128, 128),
(2048, 2048, 2048),
id="qwen35_08b_tp1",
),
pytest.param(
"gdn_attention",
4,
1536,
3,
(4, 128, 128),
(512, 512, 512),
id="qwen35_08b_tp4",
),
# Qwen/Qwen3.5-4B (GDN, asymmetric: num_v=32, num_k=16, K:V=1:2)
# key_dim=2048, value_dim=4096, conv_dim=8192
pytest.param(
"gdn_attention",
1,
8192,
3,
(32, 128, 128),
(2048, 2048, 4096),
id="qwen35_4b_tp1",
),
# Qwen/Qwen3.5-27B (GDN, asymmetric: num_v=48, num_k=16, K:V=1:3)
# key_dim=2048, value_dim=6144, conv_dim=10240
pytest.param(
"gdn_attention",
1,
10240,
3,
(48, 128, 128),
(2048, 2048, 6144),
id="qwen35_27b_tp1",
),
pytest.param(
"gdn_attention",
8,
1280,
3,
(6, 128, 128),
(256, 256, 768),
id="qwen35_27b_tp8",
),
],
)
def test_derive_mamba_conv_split(
monkeypatch,
mamba_type,
local_tp,
conv_dim_local,
conv_rows,
temporal_shape,
expected_proj_dims,
):
"""Parametrized test for derive_mamba_conv_split with real model configs.
Values generated by verify_conv_split.py which loads HuggingFace configs
and calls vLLM's derive_mamba_conv_split directly.
"""
from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import (
derive_mamba_conv_split,
)
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.kv_cache_interface import MambaSpec
_TYPE_MAP = {
"mamba2": MambaAttentionBackendEnum.MAMBA2,
"gdn_attention": MambaAttentionBackendEnum.GDN_ATTN,
}
mamba_type_enum = _TYPE_MAP[mamba_type]
monkeypatch.setenv("VLLM_SSM_CONV_STATE_LAYOUT", "DS")
spec = MambaSpec(
block_size=64,
shapes=((conv_dim_local, conv_rows), temporal_shape),
dtypes=(torch.bfloat16, torch.bfloat16),
mamba_type=mamba_type_enum,
)
out = derive_mamba_conv_split(spec, local_tp=local_tp)
assert out.local_proj_dims == expected_proj_dims
assert out.conv_rows == conv_rows
@pytest.mark.cpu_test
@pytest.mark.parametrize(
"mamba_enabled,swa_enabled,"