From 24337fb860a89f2087df27b02b7c9620a7fd0093 Mon Sep 17 00:00:00 2001 From: zhanqiuhu <49648934+ZhanqiuHu@users.noreply.github.com> Date: Thu, 14 May 2026 10:33:01 -0400 Subject: [PATCH] PD disagg with NIXL Connector: GDN support (Qwen3.5) (#41869) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Zhanqiu Hu Co-authored-by: Nicolò Lucchesi --- .buildkite/test_areas/disaggregated.yaml | 2 +- .../config_sweep_accuracy_test.sh | 3 + .../nixl_integration/test_accuracy.py | 1 + .../unit/test_nixl_connector_hma.py | 126 ++++++++++++ .../kv_connector/v1/nixl/worker.py | 16 +- .../v1/ssm_conv_transfer_utils.py | 179 +++++++++++------- 6 files changed, 244 insertions(+), 83 deletions(-) diff --git a/.buildkite/test_areas/disaggregated.yaml b/.buildkite/test_areas/disaggregated.yaml index e68b9e1add..e3ba160447 100644 --- a/.buildkite/test_areas/disaggregated.yaml +++ b/.buildkite/test_areas/disaggregated.yaml @@ -51,7 +51,7 @@ steps: - label: Hybrid SSM NixlConnector PD accuracy tests (4 GPUs) key: hybrid-ssm-nixlconnector-pd-accuracy-tests-4-gpus - timeout_in_minutes: 20 + timeout_in_minutes: 25 working_dir: "/vllm-workspace/tests" num_devices: 4 source_file_dependencies: diff --git a/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh b/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh index 9bc0f3135e..c3056cafaa 100755 --- a/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh +++ b/tests/v1/kv_connector/nixl_integration/config_sweep_accuracy_test.sh @@ -21,6 +21,9 @@ hybrid_ssm_configs=( "VLLM_SSM_CONV_STATE_LAYOUT=DS ENABLE_HMA_FLAG=1 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=ibm-granite/granite-4.0-h-tiny VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192,--trust-remote-code" # TODO: (NickLucche) Address async scheduling issue with TP>1 separately as this may impact other models. "VLLM_SSM_CONV_STATE_LAYOUT=DS ENABLE_HMA_FLAG=1 PREFILLER_TP_SIZE=2 DECODER_TP_SIZE=2 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=ibm-granite/granite-4.0-h-tiny VLLM_SERVE_EXTRA_ARGS=--max-model-len,8192,--trust-remote-code,--no-async-scheduling" + # GDN (Qwen3.5) + "VLLM_SSM_CONV_STATE_LAYOUT=DS ENABLE_HMA_FLAG=1 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B" + "VLLM_SSM_CONV_STATE_LAYOUT=DS ENABLE_HMA_FLAG=1 PREFILLER_TP_SIZE=1 DECODER_TP_SIZE=2 GPU_MEMORY_UTILIZATION=0.8 MODEL_NAMES=Qwen/Qwen3.5-0.8B VLLM_SERVE_EXTRA_ARGS=--no-async-scheduling" ) sw_attn_configs=( # NOTE: gemma3 does not work with FlashInfer diff --git a/tests/v1/kv_connector/nixl_integration/test_accuracy.py b/tests/v1/kv_connector/nixl_integration/test_accuracy.py index 23ea0d2611..debf3cc421 100644 --- a/tests/v1/kv_connector/nixl_integration/test_accuracy.py +++ b/tests/v1/kv_connector/nixl_integration/test_accuracy.py @@ -20,6 +20,7 @@ EXPECTED_VALUES = { "google/gemma-3-4b-it": 0.74, "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8": 0.84, "ibm-granite/granite-4.0-h-tiny": 0.80, + "Qwen/Qwen3.5-0.8B": 0.33, } SIMPLE_PROMPT = ( diff --git a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py index 2edd6a8720..c80dbfc628 100644 --- a/tests/v1/kv_connector/unit/test_nixl_connector_hma.py +++ b/tests/v1/kv_connector/unit/test_nixl_connector_hma.py @@ -742,6 +742,132 @@ def test_compute_physical_blocks_per_logical(ssm_sizes, block_len, expected_rati assert compute_physical_blocks_per_logical(ssm_sizes, block_len) == expected_ratio +@pytest.mark.cpu_test +@pytest.mark.parametrize( + "mamba_type,local_tp,conv_dim_local,conv_rows,temporal_shape,expected_proj_dims", + [ + # nvidia/Nemotron-H-8B-Base-8K (Mamba2) + # mamba_num_heads=128, head_dim=64, n_groups=8, ssm_state_size=128 + pytest.param( + "mamba2", + 1, + 10240, + 3, + (128, 64, 128), + (8192, 1024, 1024), + id="nemotron_h_8b_tp1", + ), + pytest.param( + "mamba2", + 4, + 2560, + 3, + (32, 64, 128), + (2048, 256, 256), + id="nemotron_h_8b_tp4", + ), + # nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B (Mamba2) + # mamba_num_heads=64, head_dim=64, n_groups=8, ssm_state_size=128 + pytest.param( + "mamba2", + 1, + 6144, + 3, + (64, 64, 128), + (4096, 1024, 1024), + id="nemotron_nano_30b_tp1", + ), + # Qwen/Qwen3.5-0.8B (GDN, symmetric: num_v=num_k=16) + # key_dim=2048, value_dim=2048, conv_dim=6144 + pytest.param( + "gdn_attention", + 1, + 6144, + 3, + (16, 128, 128), + (2048, 2048, 2048), + id="qwen35_08b_tp1", + ), + pytest.param( + "gdn_attention", + 4, + 1536, + 3, + (4, 128, 128), + (512, 512, 512), + id="qwen35_08b_tp4", + ), + # Qwen/Qwen3.5-4B (GDN, asymmetric: num_v=32, num_k=16, K:V=1:2) + # key_dim=2048, value_dim=4096, conv_dim=8192 + pytest.param( + "gdn_attention", + 1, + 8192, + 3, + (32, 128, 128), + (2048, 2048, 4096), + id="qwen35_4b_tp1", + ), + # Qwen/Qwen3.5-27B (GDN, asymmetric: num_v=48, num_k=16, K:V=1:3) + # key_dim=2048, value_dim=6144, conv_dim=10240 + pytest.param( + "gdn_attention", + 1, + 10240, + 3, + (48, 128, 128), + (2048, 2048, 6144), + id="qwen35_27b_tp1", + ), + pytest.param( + "gdn_attention", + 8, + 1280, + 3, + (6, 128, 128), + (256, 256, 768), + id="qwen35_27b_tp8", + ), + ], +) +def test_derive_mamba_conv_split( + monkeypatch, + mamba_type, + local_tp, + conv_dim_local, + conv_rows, + temporal_shape, + expected_proj_dims, +): + """Parametrized test for derive_mamba_conv_split with real model configs. + + Values generated by verify_conv_split.py which loads HuggingFace configs + and calls vLLM's derive_mamba_conv_split directly. + """ + from vllm.distributed.kv_transfer.kv_connector.v1.ssm_conv_transfer_utils import ( + derive_mamba_conv_split, + ) + from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum + from vllm.v1.kv_cache_interface import MambaSpec + + _TYPE_MAP = { + "mamba2": MambaAttentionBackendEnum.MAMBA2, + "gdn_attention": MambaAttentionBackendEnum.GDN_ATTN, + } + mamba_type_enum = _TYPE_MAP[mamba_type] + + monkeypatch.setenv("VLLM_SSM_CONV_STATE_LAYOUT", "DS") + spec = MambaSpec( + block_size=64, + shapes=((conv_dim_local, conv_rows), temporal_shape), + dtypes=(torch.bfloat16, torch.bfloat16), + mamba_type=mamba_type_enum, + ) + out = derive_mamba_conv_split(spec, local_tp=local_tp) + assert out.local_proj_dims == expected_proj_dims + assert out.conv_rows == conv_rows + + @pytest.mark.cpu_test @pytest.mark.parametrize( "mamba_enabled,swa_enabled," diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py index a5f16aec57..ea8b46c28f 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/nixl/worker.py @@ -1057,8 +1057,8 @@ class NixlConnectorWorker: tp_ratio: int, transfer_info: EngineTransferInfo, ) -> list[tuple[int, int, int]]: - """Build 4 remote desc regions (x, B, C, ssm) per layer for - the 3-read transfer. For hetero-TP, each D rank reads only its + """Build 4 remote desc regions (proj0, proj1, proj2, ssm) per layer + for the 3-read transfer. For hetero-TP, each D rank reads only its sub-projection slice from the P rank.""" assert self._conv_decomp is not None effective_ratio = max(tp_ratio, 1) @@ -1067,20 +1067,10 @@ class NixlConnectorWorker: local_offset = self.tp_rank % effective_ratio conv_size_remote = nixl_agent_meta.ssm_sizes[0] + conv_offsets = self._conv_decomp.remote_conv_offsets(local_offset, tp_ratio) if tp_ratio >= 1: - # D_TP >= P_TP: P page is larger, D reads its slice. - conv_offsets = self._conv_decomp.remote_conv_offsets( - local_offset, effective_ratio - ) ssm_read_size = self._mamba_ssm_size[1] else: - # NOTE (ZhanqiuHu): tp_ratio < 0 means P_TP > D_TP, so P pages - # are smaller than D's. self._conv_decomp has D-sized dimensions, - # but we need P-sized offsets. Scale down by |tp_ratio|. - abs_ratio = -tp_ratio - xb_p = self._conv_decomp.x_bytes // abs_ratio - bb_p = self._conv_decomp.b_bytes // abs_ratio - conv_offsets = [(0, xb_p), (xb_p, bb_p), (xb_p + bb_p, bb_p)] ssm_read_size = nixl_agent_meta.ssm_sizes[1] remote_physical_per_logical = transfer_info.remote_physical_blocks_per_logical diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py b/vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py index 4ec5613355..2a5510656b 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/ssm_conv_transfer_utils.py @@ -2,9 +2,14 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """Mamba conv-state sub-projection decomposition for the 3-read transfer. -With DS conv state layout (dim, state_len), x/B/C sub-projections are -contiguous in memory. Each D rank reads its x, B, C slices via 3 -separate RDMA transfers — no P-side permutation needed. +With DS conv state layout (dim, state_len), sub-projections are +contiguous in memory. Each D rank reads its slices via 3 separate +RDMA transfers — no P-side permutation needed. + +Supported model types: + - Mamba2: conv = [x, B, C], temporal = (num_heads, head_dim) + - GDN (Gated Delta Net): conv = [Q, K, V] (dim(Q)==dim(K)), + temporal = (num_v_heads, v_dim, k_dim) """ import math @@ -19,97 +24,109 @@ from vllm.v1.kv_cache_interface import MambaSpec @dataclass(frozen=True) class MambaConvSplitInfo: - """Per-rank byte sizes of x, B, C sub-projections in the Mamba conv state. + """Per-rank byte sizes of the 3 conv sub-projections. Used by both P and D sides for NIXL descriptor registration. All fields are LOCAL to this engine's TP (already divided by TP size). - DS memory layout within one page (contiguous in memory): - |--- x (x_local * conv_rows) ---|- B (b_local * conv_rows) -|- C -| + DS memory layout within one page (contiguous): + Mamba2: |-- x --|- B -|- C -| (B == C) + GDN: |- Q -|- K -|-- V --| (dim(Q)==dim(K), V may differ) """ conv_rows: int # conv_kernel - 1 (typically 3) - x_local: int # intermediate_size / TP (columns for x) - b_local: int # groups_ss / TP (columns for B; C is same size) + local_proj_dims: tuple[int, int, int] # per-rank column counts per sub-proj conv_dtype_size: int # bytes per element (e.g. 2 for float16) ssm_sizes: tuple[int, int] # (conv_state_bytes, ssm_state_bytes) @property - def conv_dim_local(self) -> int: - """Total conv columns per rank: x + B + C.""" - return self.x_local + 2 * self.b_local + def local_conv_dim(self) -> int: + """Total conv columns per rank.""" + return sum(self.local_proj_dims) @property - def x_bytes(self) -> int: - """Byte size of the x sub-projection for one rank.""" - return self.x_local * self.conv_rows * self.conv_dtype_size - - @property - def b_bytes(self) -> int: - """Byte size of the B (or C) sub-projection for one rank.""" - return self.b_local * self.conv_rows * self.conv_dtype_size + def proj_bytes(self) -> tuple[int, int, int]: + """Byte sizes of the 3 sub-projections for one rank.""" + row_bytes = self.conv_rows * self.conv_dtype_size + return tuple(d * row_bytes for d in self.local_proj_dims) # type: ignore[return-value] @property def local_conv_offsets(self) -> list[tuple[int, int]]: - """(byte_offset, byte_size) of x, B, C within this engine's page. + """(byte_offset, byte_size) of each sub-projection within this + engine's page. Used by both P and D for local descriptor registration. """ - xb = self.x_bytes - bb = self.b_bytes - return [(0, xb), (xb, bb), (xb + bb, bb)] + conv0, conv1, conv2 = self.proj_bytes + return [(0, conv0), (conv0, conv1), (conv0 + conv1, conv2)] def remote_conv_offsets( self, local_rank_offset: int, tp_ratio: int ) -> list[tuple[int, int]]: - """(byte_offset, byte_size) of this D rank's x, B, C slice within - one P page. + """(byte_offset, byte_size) of this D rank's sub-projection slices + within one P page. Used by D side only, during remote descriptor registration. Args: local_rank_offset: which slice this D rank reads. - tp_ratio > 0: tp_rank % tp_ratio (selects slice of P's page). - tp_ratio < 0: always 0 (read P's full page). - tp_ratio: effective ratio (>= 1 when D_TP > P_TP, 1 when - P_TP > D_TP since each P rank is read in full). + tp_ratio: signed TP ratio. + >= 1: D_TP >= P_TP — P page is larger, D reads its slice. + < 0: P_TP > D_TP — P pages are smaller, D reads entire + P page. Local dims are scaled down by |tp_ratio| + to get P-sized offsets. """ - xb = self.x_bytes - bb = self.b_bytes - xr = xb * tp_ratio # full remote x section in bytes - br = bb * tp_ratio # full remote B section in bytes - return [ - (local_rank_offset * xb, xb), - (xr + local_rank_offset * bb, bb), - (xr + br + local_rank_offset * bb, bb), - ] + conv0, conv1, conv2 = self.proj_bytes + if tp_ratio >= 1: + remote_conv0 = conv0 * tp_ratio + remote_conv1 = conv1 * tp_ratio + return [ + (local_rank_offset * conv0, conv0), + (remote_conv0 + local_rank_offset * conv1, conv1), + (remote_conv0 + remote_conv1 + local_rank_offset * conv2, conv2), + ] + else: + # NOTE (ZhanqiuHu): tp_ratio < 0 means P_TP > D_TP, so P pages + # are smaller than D's. Local dims are D-sized, but we need + # P-sized offsets. Scale down by |tp_ratio|. + abs_ratio = -tp_ratio + remote_conv0 = conv0 // abs_ratio + remote_conv1 = conv1 // abs_ratio + remote_conv2 = conv2 // abs_ratio + return [ + (0, remote_conv0), + (remote_conv0, remote_conv1), + (remote_conv0 + remote_conv1, remote_conv2), + ] def derive_mamba_conv_split( mamba_spec: MambaSpec, local_tp: int, ) -> MambaConvSplitInfo: - """Derive per-rank x/B/C byte sizes from a MambaSpec. + """Derive per-rank sub-projection byte sizes from a MambaSpec. Called once at init on both P and D. Decomposes the conv dimension - (= intermediate_size + 2 * groups_ss) into its x, B, C parts. + into its sub-projection parts based on the model type. Args: mamba_spec: MambaSpec whose shapes are: shapes[0] = conv state: (conv_dim_local, conv_rows) in DS layout. - shapes[1] = SSM temporal: (local_num_heads, head_dim). + shapes[1] = temporal state (model-specific shape). local_tp: this engine's tensor-parallel size. Returns: - MambaConvSplitInfo with per-rank x_local, b_local, conv_rows, + MambaConvSplitInfo with per-rank sub-projection dims, conv_rows, conv_dtype_size, and ssm_sizes (conv_state_bytes, ssm_state_bytes). """ - if mamba_spec.mamba_type != MambaAttentionBackendEnum.MAMBA2: + _supported = ( + MambaAttentionBackendEnum.MAMBA2, + MambaAttentionBackendEnum.GDN_ATTN, + ) + if mamba_spec.mamba_type not in _supported: raise NotImplementedError( - f"3-read conv transfer only supports Mamba2 models, " - f"got mamba_type={mamba_spec.mamba_type!r}. " - f"Mamba1 SSM temporal shape is (intermediate_size // tp, state_size) " - f"which cannot be used to reconstruct intermediate_size." + f"3-read conv transfer only supports Mamba2 and GDN models, " + f"got mamba_type={mamba_spec.mamba_type!r}." ) conv_shape = mamba_spec.shapes[0] @@ -121,29 +138,10 @@ def derive_mamba_conv_split( local_conv_dim = conv_shape[0] # DS: (conv_dim_local, conv_rows) conv_rows = conv_shape[1] - # NOTE (ZhanqiuHu): intermediate_size (= global x dim) is not stored - # in MambaSpec, so we reconstruct it from the SSM temporal state shape: - # shapes[1] = (local_num_heads, head_dim), already divided by TP. - head_dim = mamba_spec.shapes[1][1] - local_num_heads = mamba_spec.shapes[1][0] - intermediate_size = local_num_heads * local_tp * head_dim - - # NOTE (ZhanqiuHu): global conv dim = intermediate_size + 2 * groups_ss, - # where groups_ss is the B (= C) dimension. B and C are always the same - # size, so we recover groups_ss from the remainder after subtracting x. - remainder = local_conv_dim * local_tp - intermediate_size - assert remainder > 0 and remainder % 2 == 0, ( - f"Conv dim ({local_conv_dim}*tp={local_tp}) doesn't decompose into " - f"intermediate_size={intermediate_size} + 2*groups_ss. " - f"remainder={remainder}" - ) - groups_ss = remainder // 2 - conv_dtype_size = torch.tensor( [], dtype=mamba_spec.dtypes[0], # type: ignore[misc] ).element_size() - ssm_dtype_size = torch.tensor( [], dtype=mamba_spec.dtypes[1], # type: ignore[misc] @@ -151,11 +149,54 @@ def derive_mamba_conv_split( conv_state_bytes = torch.Size(mamba_spec.shapes[0]).numel() * conv_dtype_size ssm_state_bytes = torch.Size(mamba_spec.shapes[1]).numel() * ssm_dtype_size - # Divide by TP to get per-rank column counts. + if mamba_spec.mamba_type == MambaAttentionBackendEnum.MAMBA2: + # NOTE (ZhanqiuHu): intermediate_size (= global x dim) is not stored + # in MambaSpec, so we reconstruct it from the SSM temporal state shape: + # shapes[1] = (local_num_heads, head_dim), already divided by TP. + head_dim = mamba_spec.shapes[1][1] + local_num_heads = mamba_spec.shapes[1][0] + intermediate_size = local_num_heads * local_tp * head_dim + + # NOTE (ZhanqiuHu): global conv dim = intermediate_size + 2 * groups_ss, + # where groups_ss is the B (= C) dimension. B and C are always the same + # size, so we recover groups_ss from the remainder after subtracting x. + remainder = local_conv_dim * local_tp - intermediate_size + assert remainder > 0 and remainder % 2 == 0, ( + f"Conv dim ({local_conv_dim}*tp={local_tp}) doesn't decompose " + f"into intermediate_size={intermediate_size} + 2*groups_ss. " + f"remainder={remainder}" + ) + groups_ss = remainder // 2 + + # Divide by TP to get per-rank column counts. + x_local = intermediate_size // local_tp + b_local = groups_ss // local_tp + local_proj_dims = (x_local, b_local, b_local) + elif mamba_spec.mamba_type == MambaAttentionBackendEnum.GDN_ATTN: + # GDN: conv = [Q, K, V] where dim(Q) == dim(K) == key_dim. + # conv_dim = key_dim*2 + value_dim (all global, divided by TP). + # Temporal state shape is (num_v_heads/TP, head_v_dim, head_k_dim). + temporal_shape = mamba_spec.shapes[1] + num_v_heads_local = temporal_shape[0] + head_v_dim = temporal_shape[1] + value_dim_local = num_v_heads_local * head_v_dim + + remainder = local_conv_dim - value_dim_local + assert remainder > 0 and remainder % 2 == 0, ( + f"GDN conv dim ({local_conv_dim}) doesn't decompose into " + f"2*key_dim_local + value_dim_local={value_dim_local}. " + f"remainder={remainder}" + ) + key_dim_local = remainder // 2 + local_proj_dims = (key_dim_local, key_dim_local, value_dim_local) + else: + raise NotImplementedError( + f"Conv split not supported for mamba_type={mamba_spec.mamba_type!r}" + ) + return MambaConvSplitInfo( conv_rows=conv_rows, - x_local=intermediate_size // local_tp, - b_local=groups_ss // local_tp, + local_proj_dims=local_proj_dims, conv_dtype_size=conv_dtype_size, ssm_sizes=(conv_state_bytes, ssm_state_bytes), )